level=info ts=2026-05-12T21:23:14.061744785Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-12T21:23:14.061769455Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-12T21:23:14.061824716Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-12T21:23:14.063554731Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-12T21:23:14.064031679Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:23:14.064168382Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.72 inf=eth0
level=info ts=2026-05-12T21:23:14.064739391Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-0eb69d50
ts=2026-05-12T21:23:14.06484045Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-12T21:23:14.064852778Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.72 advertisePort=7946
level=debug ts=2026-05-12T21:23:14.064886347Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.72 advertisePort=7946
level=info ts=2026-05-12T21:23:14.065020569Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-12T21:23:14.070853929Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:23:14.070922063Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-05-12T21:23:14.072747917Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:23:14.072781825Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:23:14.072806635Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.787838ms
level=error ts=2026-05-12T21:23:14.07282342Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-12T21:23:14.072837087Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-12T21:23:14.073508853Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-12T21:23:14.073779324Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:23:14.074243759Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-12T21:23:14.074271829Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-12T21:23:14.074268339Z
level=debug ts=2026-05-12T21:23:14.074286557Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-12T21:23:14.074352037Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-12T21:23:14.074379893Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-12T21:23:14.074388165Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-12T21:23:14.074386657Z
level=debug ts=2026-05-12T21:23:14.074407313Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-05-12T21:23:14.078928494Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:14.078944011Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:23:14.168274561Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:23:14.168429171Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.72 inf=eth0
level=info ts=2026-05-12T21:23:14.18534617Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-12T21:23:14.185360564Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:23:14.185357911Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-12T21:23:14.185409906Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:23:14.185410256Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-12T21:23:14.185652012Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-05-12T21:23:14.185673135Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-12T21:23:14.185790406Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-12T21:23:14.185796819Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-12T21:23:14.185826592Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-12T21:23:14.185828203Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-12T21:23:14.186236479Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-12T21:23:14.186240825Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-12T21:23:14.186248083Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-12T21:23:14.18625059Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-05-12T21:23:14.186256137Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-12T21:23:14.186264236Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-12T21:23:14.186307891Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-12T21:23:14.186353876Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-12T21:23:14.186360531Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-12T21:23:14.186365102Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-12T21:23:14.186459679Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:23:14.186471105Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-12T21:23:14.186474311Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-05-12T21:23:14.186477239Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-12T21:23:14.186502576Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-12T21:23:14.186510352Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:23:14.186564472Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=26.379µs success=true
level=info ts=2026-05-12T21:23:14.186577853Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-12T21:23:14.186631082Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=131.527µs errors=false
level=info ts=2026-05-12T21:23:14.186643225Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-12T21:23:14.186778186Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=278.311µs errors=false
level=info ts=2026-05-12T21:23:14.186790727Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-12T21:23:14.186798855Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=298.952µs
level=info ts=2026-05-12T21:23:14.186820551Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-12T21:23:14.186856641Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-12T21:23:14.186849786Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=452.387442ms
level=info ts=2026-05-12T21:23:14.186880252Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-12T21:23:14.186934537Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-12T21:23:14.186947081Z caller=loki.go:615 msg="Loki started" startup_time=189.175117ms
level=debug ts=2026-05-12T21:23:14.18698205Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-12T21:23:14.186990224Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-12T21:23:14.187027857Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-12T21:23:15.916752687Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:15.916774235Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:23:18.820343176Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:18.820367407Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:23:19.188033769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:23:24.187410089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:23:25.441556124Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:25.441578462Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:23:29.064419093Z caller=logging.go:137 msg="GET /ready (503) 104.423µs"
level=warn ts=2026-05-12T21:23:29.187872587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:23:34.187367939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:23:34.821388158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.73:7946"
ts=2026-05-12T21:23:34.823459994Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.61:7946: dial tcp 10.128.0.61:7946: connect: connection refused"
level=info ts=2026-05-12T21:23:34.823473907Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.750629767s
level=debug ts=2026-05-12T21:23:39.062779669Z caller=logging.go:137 msg="GET /ready (503) 114.906µs"
level=warn ts=2026-05-12T21:23:39.187654188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:23:44.18760014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:23:44.187618061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:23:44.187632926Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-12T21:23:44.187642808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:23:49.187408146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:23:50.160711736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:23:54.187590632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:23:59.187991587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:24:04.187097114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:24:09.187950269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:24:11.134995252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:24:14.072492134Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:24:14.07463015Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:24:14.187744138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:24:14.187772942Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:24:14.187782914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:24:14.187798605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:24:15.048942462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:24:19.18767212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:24:24.188010782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:24:25.32555255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:24:25.325612157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:24:25.335504258Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:24:29.187497587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:24:32.758391842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:24:34.187735305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:24:39.18806214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:24:41.137743877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T21:24:43.03763826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:24:43.045268677Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:24:44.187757153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:24:44.187785094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:24:44.187792891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:24:44.187811735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:24:49.187350252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:24:51.620659689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:24:54.187714711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:24:54.810423084Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:24:59.188095021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:25:04.187813088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:25:05.068297422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59724"
level=warn ts=2026-05-12T21:25:09.187761066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:25:11.1397122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:25:13.778194905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:25:13.787507245Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:25:14.07222562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:25:14.074327594Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:25:14.187876953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:25:14.187906784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:25:14.187914391Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:25:14.187906422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:25:14.645560779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:25:19.187799714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:25:22.585235866Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:25:24.187963906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:25:24.510016599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39972"
level=warn ts=2026-05-12T21:25:29.187314674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:25:34.187199156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:25:39.187535472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:25:41.141545396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T21:25:44.187676219Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:25:44.187712243Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:25:44.187723604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:25:44.187721376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:25:44.78349689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60678"
level=warn ts=2026-05-12T21:25:49.18719064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:25:49.756638243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:25:49.758329693Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:25:54.187384826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:25:54.723603861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:25:54.732721306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:25:56.290388564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:25:59.187193886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:26:04.187855884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:26:09.188061987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:26:11.145079406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:26:14.073826572Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:26:14.073832278Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:26:14.187545601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:26:14.187565487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:26:14.187582342Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:26:14.187593886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:26:19.187984395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:26:24.18751175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:26:28.541946303Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:26:29.187274083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:26:34.187548308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:26:35.168339848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T21:26:38.344731625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:26:38.356442241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:26:39.187563403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:26:41.147359755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:26:44.187558155Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:26:44.187564791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:26:44.187587409Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:26:44.187595763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:26:49.187893997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:26:54.187044429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:26:54.203248757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:26:59.187261134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:27:04.187719217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:27:09.187530728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:27:11.148841948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:27:14.072806247Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:27:14.073839997Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T21:27:14.187295287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:27:14.187338645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:27:14.187363763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:27:14.187372581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:27:19.187392936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:27:21.704102473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:27:23.064496069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:27:23.073244748Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:27:23.216963468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:27:24.187754456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:27:24.522093759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53018"
level=warn ts=2026-05-12T21:27:29.187470151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:27:34.186994974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:27:39.187121546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:27:41.150031286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:27:44.187040341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:27:44.187087703Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:27:44.187096122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:27:44.187087848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:27:49.187945047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:27:49.765305341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:27:54.18801539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:27:54.524792861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:46614"
level=warn ts=2026-05-12T21:27:59.187924737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:28:04.187147311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:28:09.187360033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:11.151286744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:28:14.07280837Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:28:14.073847388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:28:14.18737288Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:28:14.187396125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:28:14.187410306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:14.187421149Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:28:14.187428964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:28:14.187542831Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-12T21:28:14.187554106Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:28:14.187576079Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.571µs success=true
level=debug ts=2026-05-12T21:28:14.187815349Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-12T21:28:16.219868843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:28:18.543333Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-12T21:28:19.187757903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:20.026566028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:28:21.35067883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:28:21.359133027Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:28:22.898427192Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-12T21:28:24.187818012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:24.527892708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34148"
level=debug ts=2026-05-12T21:28:27.252849047Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:28:29.187289063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:31.607353822Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:28:34.187170609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:35.962380819Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-12T21:28:39.188046887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:40.317193259Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-12T21:28:41.152461186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:28:44.187824631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:28:44.187851572Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:28:44.187859928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:28:44.187838679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:44.672473457Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T21:28:49.026610423Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:28:49.187991648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:53.382015288Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=warn ts=2026-05-12T21:28:54.18804083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:54.531576344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49098"
level=debug ts=2026-05-12T21:28:54.871942759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:28:57.737169634Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-12T21:28:59.187581855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:02.091292039Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T21:29:04.187868176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:29:05.077775655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37282"
level=info ts=2026-05-12T21:29:05.558191036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:29:05.570782806Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:29:06.446198885Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-12T21:29:09.187189582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:10.800944994Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-12T21:29:10.800976148Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
ts=2026-05-12T21:29:11.153499293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:29:14.072457262Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:29:14.07455622Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T21:29:14.187029567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:29:14.187065393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:29:14.187087858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:29:14.187095084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:29:15.155816195Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=warn ts=2026-05-12T21:29:19.187889545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:19.510859313Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-12T21:29:23.865419691Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-12T21:29:24.187481165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:24.492150219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T21:29:24.535089706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34692"
level=debug ts=2026-05-12T21:29:28.223455731Z caller=checkpoint.go:372 msg="writing series" size="4.4 MB"
level=debug ts=2026-05-12T21:29:28.223482369Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=4
level=warn ts=2026-05-12T21:29:29.188105518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:32.575131722Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-12T21:29:32.801671323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:29:34.187375357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:36.930382124Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:29:39.187847563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:29:41.154366723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:29:41.285509849Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-12T21:29:44.187734728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:29:44.187761409Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:29:44.187769561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:29:44.187782339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:45.639920291Z caller=checkpoint.go:372 msg="writing series" size="608 kB"
level=warn ts=2026-05-12T21:29:49.187938501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:49.994688597Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-12T21:29:50.625394322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:29:54.187814149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:54.349554211Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
ts=2026-05-12T21:29:54.538219998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48836"
level=info ts=2026-05-12T21:29:55.749001313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:29:55.757640909Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:29:58.704150318Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=warn ts=2026-05-12T21:29:59.187733387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:03.059692531Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T21:30:04.187786356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:05.080626988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59190"
level=debug ts=2026-05-12T21:30:07.413987133Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-12T21:30:09.187732656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:11.156582838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:30:11.770049421Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T21:30:11.770089921Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=10
level=debug ts=2026-05-12T21:30:14.073776434Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:30:14.07482127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:30:14.076149106Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T21:30:14.076196227Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976245 n=1
level=debug ts=2026-05-12T21:30:14.076238156Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976245 n=1
level=info ts=2026-05-12T21:30:14.187342843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:30:14.187370065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:30:14.187378076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:30:14.187374568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:16.123492192Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:30:19.187694365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:20.478351321Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-12T21:30:22.420966451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:30:24.187924588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:24.540960936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:33818"
level=debug ts=2026-05-12T21:30:24.833527897Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-12T21:30:29.18771198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:29.187919292Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=debug ts=2026-05-12T21:30:33.542755306Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-12T21:30:34.187345403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:35.082010243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44418"
level=debug ts=2026-05-12T21:30:37.898129653Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T21:30:39.187561895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:41.160056238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:30:42.253145014Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-12T21:30:43.526853331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:30:43.528668885Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:30:44.187850054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:30:44.18787908Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:30:44.187861197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:30:44.187886779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:30:45.547531192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:30:45.554430757Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:30:45.554453655Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:30:45.554475213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:30:45.561063247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:30:46.607517003Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T21:30:49.187353662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:50.482713231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:30:50.962912265Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-12T21:30:54.18704753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:55.317437867Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-12T21:30:55.697600189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:30:55.706906151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:30:59.187735894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:59.672184846Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-12T21:31:04.027059328Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-12T21:31:04.027097244Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-12T21:31:04.187715365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:31:05.083208875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36948"
level=debug ts=2026-05-12T21:31:08.381576635Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-12T21:31:09.187319902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:31:11.162587341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:31:12.737196231Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T21:31:14.072888643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:31:14.073939413Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:31:14.187573473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:31:14.187574053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:14.187597805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:31:14.187605979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:31:17.091916078Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-12T21:31:19.187765182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:20.159866708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:31:21.44665251Z caller=checkpoint.go:372 msg="writing series" size="395 kB"
level=warn ts=2026-05-12T21:31:24.187917525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:25.801882256Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-12T21:31:28.387018894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:31:28.396854252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:31:29.187744277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:30.156155517Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T21:31:32.438585437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:31:34.187775503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:34.511119654Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-12T21:31:38.8655664Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-05-12T21:31:39.187988731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:31:41.164302797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:31:43.221001082Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-12T21:31:44.187659389Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:31:44.187669216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:44.187695565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:31:44.187703792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:31:44.8058014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46476"
level=debug ts=2026-05-12T21:31:47.210608257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:31:47.575186571Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=debug ts=2026-05-12T21:31:47.575210769Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-12T21:31:49.18803536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:51.930499708Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:31:54.187843149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:56.285387482Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-12T21:31:59.187860399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:00.639938662Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-12T21:32:04.187911321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:04.994467501Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-12T21:32:09.18801509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:09.350216489Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-12T21:32:11.165589834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:32:12.427064987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:32:13.704128121Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-12T21:32:14.073208396Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:32:14.074254419Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T21:32:14.187988757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:32:14.188022293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:32:14.188048969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:32:14.188058897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:32:16.51971985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:32:16.531538824Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:32:18.059116428Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:32:19.187895634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:19.989191726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:32:19.98924731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:32:19.997197051Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:32:22.41381602Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-12T21:32:24.187967671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:26.769461407Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-12T21:32:29.187686434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:31.124164307Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:32:34.187375288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:32:34.974006368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:32:34.989202157Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T21:32:35.086718571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39866"
level=debug ts=2026-05-12T21:32:35.478879982Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:32:39.187896765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:39.833759876Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
ts=2026-05-12T21:32:41.168587075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T21:32:44.187767239Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:32:44.187789006Z caller=checkpoint.go:384 msg="flushing series" totalSize="818 kB" series=12
level=debug ts=2026-05-12T21:32:44.187807792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:32:44.187818835Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:32:44.187833981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:32:44.296791291Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-12T21:32:44.300332307Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.112573785s
level=warn ts=2026-05-12T21:32:49.188056411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:50.941482418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:32:52.246207914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:32:54.188060857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:32:54.555557594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:33424"
level=info ts=2026-05-12T21:32:56.292341682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:32:56.30101499Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:32:59.187909892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:33:04.187450238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:33:09.187782853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:33:09.390783483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:33:09.401299609Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T21:33:11.171553126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:33:14.073124293Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:33:14.074155547Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:33:14.187287242Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:33:14.187286526Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:33:14.18731946Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:33:14.187330595Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:33:14.18735802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:33:14.187438492Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-12T21:33:14.187446775Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:33:14.187466364Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.631µs success=true
level=debug ts=2026-05-12T21:33:14.18766249Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-12T21:33:14.811054486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46330"
level=debug ts=2026-05-12T21:33:17.788123873Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-12T21:33:19.188052088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:21.388179468Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-12T21:33:24.186991774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:33:24.558267885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36878"
level=debug ts=2026-05-12T21:33:24.988584265Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-12T21:33:26.659416421Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:33:28.588103784Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T21:33:29.187903945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:32.188156281Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-12T21:33:34.187296186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:35.788041955Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-12T21:33:39.187613042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:39.387965334Z caller=checkpoint.go:372 msg="writing series" size="380 kB"
level=info ts=2026-05-12T21:33:40.415982052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:33:40.430233698Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T21:33:41.173274103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:33:42.988484019Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-12T21:33:44.187012825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:33:44.18705772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:33:44.187139319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:33:44.187169003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:33:44.812189578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36118"
level=debug ts=2026-05-12T21:33:46.58841639Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T21:33:47.53183107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:33:49.187504471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:50.188655372Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-12T21:33:53.788365608Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:33:54.187496536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:54.634975163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:33:57.38827732Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-12T21:33:59.187943879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:00.988064071Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T21:34:04.187305139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:04.588405392Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=debug ts=2026-05-12T21:34:04.5884303Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=info ts=2026-05-12T21:34:06.689106193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:34:06.701944973Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:34:08.188378134Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:34:09.187349019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:34:11.175787935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:34:11.788103664Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=debug ts=2026-05-12T21:34:14.073173371Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:34:14.074212089Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:34:14.187948419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:34:14.187955983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:14.18797633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:34:14.18798497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:34:14.813612667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36226"
level=debug ts=2026-05-12T21:34:15.388038803Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=debug ts=2026-05-12T21:34:18.988030586Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-12T21:34:19.187415663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:20.699551917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:34:22.591717179Z caller=checkpoint.go:372 msg="writing series" size="5.3 MB"
level=debug ts=2026-05-12T21:34:22.591745203Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=5
level=warn ts=2026-05-12T21:34:24.187186849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:26.188106017Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-12T21:34:29.187162656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:29.788111369Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=debug ts=2026-05-12T21:34:31.720721321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:34:31.722184311Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:34:33.387974042Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:34:34.187776719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:36.987873441Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=debug ts=2026-05-12T21:34:36.9878955Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T21:34:38.392851464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:34:38.399846643Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:34:39.187830197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:40.588317493Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
ts=2026-05-12T21:34:41.178443238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:34:44.187790662Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:34:44.187803376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:34:44.187826831Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:34:44.18783472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:34:44.187832618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:47.788589268Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-12T21:34:49.187705163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:51.387945519Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-12T21:34:52.840002176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:34:54.188015532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:54.988256845Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-12T21:34:58.587886977Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-12T21:34:59.187900745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:35:01.457411893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:35:01.468682361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:35:02.189562672Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-12T21:35:02.189592241Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=warn ts=2026-05-12T21:35:04.187608981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:05.787984623Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:35:09.187573463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:09.388387711Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T21:35:11.180375015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:35:12.988241057Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-12T21:35:14.073140174Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:35:14.074179052Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:35:14.187000044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:35:14.187001297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:14.187036299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:35:14.187046285Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:35:16.588663521Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-12T21:35:16.646904368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:35:19.188011342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:20.188719384Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T21:35:23.788367838Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T21:35:24.187674044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:25.648041453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:35:27.388473553Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-12T21:35:29.187751243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:30.987818378Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-12T21:35:31.779186872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:35:31.789550702Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:35:34.187617439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:34.588362247Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
ts=2026-05-12T21:35:35.100345576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47992"
level=debug ts=2026-05-12T21:35:38.18787541Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-12T21:35:39.187701632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:35:41.182790442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:35:41.788181632Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-12T21:35:44.187036627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:35:44.187082196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:35:44.187091789Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:35:44.18715926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:45.388810522Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-12T21:35:48.988611138Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-12T21:35:48.988637903Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-12T21:35:49.187762914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:35:51.046595326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:35:51.056711468Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:35:52.587987541Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T21:35:54.187283066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:56.188085997Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:35:59.187968133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:59.788659868Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-12T21:36:00.20776267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:36:03.388764294Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-12T21:36:04.187439236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:36:05.101758704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57202"
level=debug ts=2026-05-12T21:36:06.987966835Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-12T21:36:08.54804848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:36:09.187718216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:10.588142026Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-12T21:36:11.186268073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:36:14.073237857Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:36:14.074280503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:36:14.186936284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:36:14.186975926Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:36:14.18698698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:36:14.187034608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:14.1880195Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T21:36:17.788738509Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T21:36:19.187710026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:21.396492522Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-12T21:36:21.396523532Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=warn ts=2026-05-12T21:36:24.187816698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:24.98845296Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=debug ts=2026-05-12T21:36:26.29318937Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:36:28.588732419Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-12T21:36:29.188113108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:36:31.289916482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:36:31.300167545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:36:32.188606636Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T21:36:34.039212234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:36:34.047753548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:36:34.187192779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:35.788642085Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T21:36:39.187817057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:39.388671422Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
ts=2026-05-12T21:36:41.19013199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:36:42.988686799Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-12T21:36:44.187309052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:36:44.187344826Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:36:44.187338382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:36:44.187355549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:36:46.589154658Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=warn ts=2026-05-12T21:36:49.187023679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:50.188028367Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T21:36:51.374533198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:36:53.788305294Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-12T21:36:54.187794143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:36:54.582984789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:37278"
level=debug ts=2026-05-12T21:36:57.387908957Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-12T21:36:59.187697356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:00.24659578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:37:00.988061716Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-12T21:37:04.187096723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:04.58812302Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=debug ts=2026-05-12T21:37:08.188751929Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-12T21:37:09.187585812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:37:11.191805266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:37:11.788503438Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-12T21:37:11.788533064Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-05-12T21:37:14.073833132Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:37:14.074550158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:37:14.187365819Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:37:14.18740647Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:37:14.187415193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:37:14.187490987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:15.3886178Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-12T21:37:18.047296803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:37:18.057086646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:37:18.057121791Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:37:18.057156289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:37:18.06482882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:37:18.98848093Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:37:19.187203042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:21.577443151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:37:22.588482425Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:37:24.187789982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:37:24.586683212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:43054"
level=debug ts=2026-05-12T21:37:26.188064194Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=warn ts=2026-05-12T21:37:29.188055155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:37:29.211209404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:37:29.224530258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:37:29.788142175Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-12T21:37:33.388324302Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:37:34.187426256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:37:35.108868735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60786"
level=info ts=2026-05-12T21:37:35.854880854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:37:35.863836921Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:37:36.98832059Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:37:39.187696212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:40.589361395Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
ts=2026-05-12T21:37:41.193637746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:37:44.18762767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:37:44.187645948Z caller=checkpoint.go:384 msg="flushing series" totalSize="681 kB" series=8
level=debug ts=2026-05-12T21:37:44.187667306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:37:44.187677969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:37:44.187671075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:37:44.308577843Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-12T21:37:44.314159252Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.12654905s
ts=2026-05-12T21:37:44.833204412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34278"
level=debug ts=2026-05-12T21:37:46.164174134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:37:49.187220061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:37:54.1871381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:37:54.589427588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39852"
level=debug ts=2026-05-12T21:37:56.192805115Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:37:59.187197231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:38:04.187377713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:38:06.423893075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:38:06.434203329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:38:09.18792442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:38:11.196390011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:38:14.07248185Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:38:14.074596993Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:38:14.18736163Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:38:14.187394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:38:14.18740241Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:38:14.187403273Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T21:38:14.187410314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:38:14.18753367Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-12T21:38:14.187542137Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:38:14.187559217Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.442µs success=true
level=debug ts=2026-05-12T21:38:14.187768503Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-12T21:38:17.6950069Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-12T21:38:19.187636674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:20.801435047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:38:21.201323572Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-12T21:38:23.288693055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:38:23.29813277Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:38:24.187826887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:24.708149298Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-12T21:38:28.214001755Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-12T21:38:29.188327943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:31.721243178Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-12T21:38:34.187058616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:35.227657904Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=debug ts=2026-05-12T21:38:38.733602199Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=warn ts=2026-05-12T21:38:39.192287066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:38:41.203208382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:38:42.240993949Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=debug ts=2026-05-12T21:38:42.241021257Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-12T21:38:44.187842144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:38:44.190549651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:38:44.190580224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:38:44.190593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:38:44.914678697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:38:44.91670577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:38:45.746498245Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T21:38:49.188361136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:49.253652382Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T21:38:51.424440713Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:38:52.760032065Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-12T21:38:53.666270598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:38:53.685863117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:38:54.189156298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:38:54.596739347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:58338"
level=debug ts=2026-05-12T21:38:56.265994546Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-12T21:38:59.187508411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:59.774323845Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-12T21:39:03.280263617Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-12T21:39:03.80764433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:39:03.8187967Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:39:03.818828595Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:39:03.818877121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:39:03.830062157Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:39:04.189287749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:06.786250958Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:39:09.188248371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:10.293190845Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
ts=2026-05-12T21:39:11.209413738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:39:13.799043022Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=info ts=2026-05-12T21:39:14.075377216Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:39:14.080372718Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=warn ts=2026-05-12T21:39:14.188240262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:39:14.188640889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:39:14.188671825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:39:14.188693173Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:39:17.305948474Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T21:39:19.187520231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:20.816707463Z caller=checkpoint.go:372 msg="writing series" size="5.6 MB"
level=debug ts=2026-05-12T21:39:20.816734918Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.6 MB" series=11
level=info ts=2026-05-12T21:39:22.024010654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:39:22.035114052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:39:24.187347531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:24.318637424Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
ts=2026-05-12T21:39:24.600866928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54612"
level=debug ts=2026-05-12T21:39:27.311487289Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:39:27.825324436Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-05-12T21:39:29.187200972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:31.33097352Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:39:34.187632915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:34.838310044Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-12T21:39:34.838332886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-12T21:39:35.368362294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:39:35.397608984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:39:38.344046715Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-12T21:39:39.187329052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:39:41.211146585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:39:41.85090767Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=debug ts=2026-05-12T21:39:43.312414402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:39:43.55899805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:39:43.567249758Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:39:44.187104553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:39:44.187103662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:39:44.187135189Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:39:44.187145276Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:39:45.356731668Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-05-12T21:39:48.863274689Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T21:39:49.1879507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:52.370243001Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-12T21:39:52.820195828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:39:54.187090851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:55.876792822Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-12T21:39:59.18804195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:59.384204086Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-12T21:39:59.384240382Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=7
level=debug ts=2026-05-12T21:40:02.890213955Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:40:04.186999404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:06.396338314Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-12T21:40:09.187293287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:09.902771287Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T21:40:11.212749387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:40:13.409251482Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-12T21:40:14.074447223Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:40:14.075012926Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:40:14.187794852Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:40:14.187807455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:14.18783055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:40:14.1878689Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:40:16.916012758Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=info ts=2026-05-12T21:40:17.06002144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:40:17.069197656Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:40:19.187510455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:20.42195279Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=debug ts=2026-05-12T21:40:23.416632317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:40:23.928876147Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-12T21:40:24.153682035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:40:24.187970959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:40:24.607375954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:44208"
level=debug ts=2026-05-12T21:40:27.43538544Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T21:40:29.187386656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:30.94136228Z caller=checkpoint.go:372 msg="writing series" size="409 kB"
level=debug ts=2026-05-12T21:40:30.941387113Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-12T21:40:31.103994231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:40:31.11335697Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:40:34.187924521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:34.447839677Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-12T21:40:37.954648684Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=warn ts=2026-05-12T21:40:39.187044828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:40:41.214027157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:40:41.461374493Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-12T21:40:44.187548295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:40:44.187576651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:40:44.187560115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:40:44.187584202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:40:44.846215985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36320"
level=debug ts=2026-05-12T21:40:44.967639744Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-12T21:40:48.474247651Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-12T21:40:49.187534118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:51.980285316Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-12T21:40:54.187059253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:55.486821912Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-12T21:40:58.33265578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:40:58.676507318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:40:58.68958462Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:40:58.993476016Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-12T21:40:59.187563602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:02.499923342Z caller=checkpoint.go:372 msg="writing series" size="353 kB"
level=debug ts=2026-05-12T21:41:02.499949749Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-12T21:41:04.187053099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:41:05.123610915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59324"
level=debug ts=2026-05-12T21:41:06.006478793Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-12T21:41:09.18783818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:09.51274181Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
ts=2026-05-12T21:41:11.215658978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:41:11.775293583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="164 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:41:11.785391773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:41:13.019993875Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T21:41:14.073334347Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:41:14.074392429Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:41:14.18779081Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:41:14.187828405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:41:14.18782053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:41:14.187837812Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:41:16.525949409Z caller=checkpoint.go:372 msg="writing series" size="564 kB"
level=debug ts=2026-05-12T21:41:18.526913034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:41:19.187715946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:20.046322143Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-12T21:41:20.0463461Z caller=checkpoint.go:384 msg="flushing series" totalSize="23 MB" series=5
level=debug ts=2026-05-12T21:41:23.53908141Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-12T21:41:24.187139789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:27.045448355Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-12T21:41:29.187380982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:30.552204169Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-12T21:41:32.564676491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:41:34.058975754Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:41:34.187794757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:41:35.12470197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53352"
level=debug ts=2026-05-12T21:41:37.564935557Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-12T21:41:39.187514848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:41.070922748Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
ts=2026-05-12T21:41:41.217594665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=warn ts=2026-05-12T21:41:44.187002981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:41:44.187026912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:41:44.187054846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:41:44.187080647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:41:44.577525242Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
ts=2026-05-12T21:41:44.849408327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45110"
level=debug ts=2026-05-12T21:41:48.084789434Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=warn ts=2026-05-12T21:41:49.187438474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:51.591208651Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T21:41:54.043722437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:41:54.053961269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:41:54.187263958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:55.096977171Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-12T21:41:56.674347796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="164 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:41:56.683020148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:41:57.098799411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:41:58.603826258Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-12T21:41:58.603848105Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-12T21:41:59.188115555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:59.641163584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:42:02.110155412Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:42:04.187014883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:05.617101346Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-12T21:42:09.123819053Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-12T21:42:09.188040321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:42:11.219400098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:42:12.630238045Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-12T21:42:14.073837692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:42:14.074363349Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:42:14.186988431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:42:14.187030807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:42:14.187041523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:42:14.187108339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:16.136339177Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-12T21:42:19.187612828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:19.643262241Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-12T21:42:23.149625798Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:42:24.18790988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:42:24.621082662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:32898"
level=debug ts=2026-05-12T21:42:26.655825172Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=debug ts=2026-05-12T21:42:28.749203299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:42:29.187782982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:30.162418138Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-12T21:42:33.66919406Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-12T21:42:34.188211411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:37.175884627Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-12T21:42:37.17590929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-12T21:42:39.187820207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:40.681342218Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
ts=2026-05-12T21:42:41.221976585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:42:44.187860675Z caller=checkpoint.go:384 msg="flushing series" totalSize="73 kB" series=1
level=info ts=2026-05-12T21:42:44.187903642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:42:44.18789891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:44.18792992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:42:44.187944373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:42:44.192625079Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-12T21:42:44.193784571Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006055452s
level=info ts=2026-05-12T21:42:45.544758286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:42:45.553892079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:42:49.18767614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:42:50.827632745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="189 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:42:50.835645289Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:42:53.107623939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:42:53.109360512Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:42:54.187983051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:42:54.625363119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:52996"
level=warn ts=2026-05-12T21:42:59.187758747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:01.993803822Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:43:04.187220086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:43:05.128832756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39146"
level=warn ts=2026-05-12T21:43:09.187686269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:43:11.224232036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:43:14.07390978Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:43:14.073945442Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:43:14.187804984Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:43:14.187812466Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:43:14.187838673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:43:14.18784703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:43:14.18788384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:43:14.187934263Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-12T21:43:14.187941912Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:43:14.187963666Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.696µs success=true
level=debug ts=2026-05-12T21:43:14.188214981Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-12T21:43:17.694990872Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-12T21:43:19.187347557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:21.201965614Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-12T21:43:24.1878834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:24.707780028Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-12T21:43:28.214457667Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-12T21:43:29.187351248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:31.721207403Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-12T21:43:34.187271499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:43:35.016286335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:43:35.026664296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:43:35.02669318Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:43:35.026725343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:43:35.034958257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:43:35.227426225Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=debug ts=2026-05-12T21:43:37.672192907Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:43:38.734690141Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T21:43:38.73471494Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-12T21:43:39.187475371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:43:41.05293488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:43:41.060541209Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T21:43:41.225372001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:43:42.24113423Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=info ts=2026-05-12T21:43:44.186955703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:43:44.186993533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:43:44.187004291Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:43:44.187095099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:45.74749166Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T21:43:47.166730536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:43:47.174005819Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:43:49.187617687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:49.253793342Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-12T21:43:51.01903256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:43:52.760252438Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-12T21:43:54.187630348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:56.267003008Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-12T21:43:59.187974627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:59.774739942Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-12T21:44:03.280166607Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-05-12T21:44:03.280188951Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-05-12T21:44:04.187269207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:06.129476311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:44:06.786673175Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-12T21:44:08.395002237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:44:08.4085767Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:44:09.187405142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:10.292705731Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
ts=2026-05-12T21:44:11.227253021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:44:13.799439907Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=debug ts=2026-05-12T21:44:14.073108966Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:44:14.074173857Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:44:14.186935048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:44:14.186963537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:44:14.186970763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:44:14.187003776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:17.306065192Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-12T21:44:19.187818264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:20.817738607Z caller=checkpoint.go:372 msg="writing series" size="6.6 MB"
level=debug ts=2026-05-12T21:44:20.817759687Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.2 MB" series=5
level=warn ts=2026-05-12T21:44:24.187522087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:24.318961376Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-12T21:44:27.825675957Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T21:44:27.82570225Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-12T21:44:29.187731776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:31.331166264Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:44:34.187934553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:34.838211761Z caller=checkpoint.go:372 msg="writing series" size="548 kB"
level=info ts=2026-05-12T21:44:37.449541377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="236 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:44:37.46193849Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:44:38.344450882Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-12T21:44:38.413241438Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:44:39.187654834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:44:41.229758406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:44:41.850988524Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-12T21:44:43.688693161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:44:44.187877226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:44:44.187910717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:44:44.187920133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:44:44.187924241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:44:44.861680163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36388"
level=debug ts=2026-05-12T21:44:45.357602134Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-12T21:44:48.864046711Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T21:44:49.187640406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:52.371795973Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T21:44:54.187127697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:55.877587418Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-12T21:44:59.187870758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:44:59.191956879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:44:59.201156452Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:44:59.384963297Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-12T21:44:59.384987407Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=9
level=debug ts=2026-05-12T21:45:02.872465359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:45:02.890257518Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:45:04.187314068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:45:05.137528036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39010"
level=debug ts=2026-05-12T21:45:06.396633201Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-12T21:45:09.187592249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:09.902930425Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T21:45:11.231877382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:45:13.409226396Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-12T21:45:14.074182774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:45:14.074877985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=debug ts=2026-05-12T21:45:14.076186754Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T21:45:14.076254219Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976246 n=1
level=debug ts=2026-05-12T21:45:14.076307343Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976246 n=1
level=info ts=2026-05-12T21:45:14.187578334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:45:14.187580775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:14.187605122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:45:14.187612996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:45:16.915986926Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-12T21:45:19.187271268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:20.422906325Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=debug ts=2026-05-12T21:45:23.929415712Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-12T21:45:24.187245339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:27.436058717Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T21:45:29.187873656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:30.942728684Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=warn ts=2026-05-12T21:45:34.187364848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:34.448754605Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=debug ts=2026-05-12T21:45:34.448777085Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=debug ts=2026-05-12T21:45:34.973392429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:45:34.984678464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:45:35.278027088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:45:35.287291918Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:45:35.287323371Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:45:35.287367422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:45:35.295568805Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:45:37.954888577Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-12T21:45:39.188010891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:45:41.233695971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:45:41.460964442Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-12T21:45:44.187682871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:45:44.187837409Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:45:44.187865239Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:45:44.187875836Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:45:44.865625252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50926"
level=debug ts=2026-05-12T21:45:44.971417473Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-12T21:45:48.474274241Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T21:45:49.18785075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:45:50.092503428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:45:50.101979246Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:45:51.982375595Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T21:45:51.982403702Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=warn ts=2026-05-12T21:45:54.187989455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:45:55.279412221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:45:55.290969145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:45:55.487586703Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-12T21:45:58.99392017Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T21:45:59.188101047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:01.18080057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:46:02.500342484Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T21:46:04.187858461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:06.007198415Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-12T21:46:09.188047687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:09.51403865Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T21:46:09.514102081Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
ts=2026-05-12T21:46:11.236263055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:46:13.019598374Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:46:14.077219436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:46:14.077730531Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=warn ts=2026-05-12T21:46:14.187332755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:46:14.187355506Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:46:14.187384459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:46:14.187395448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:46:16.526427542Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=warn ts=2026-05-12T21:46:19.188112436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:46:19.377508812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:46:19.388465343Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:46:20.076229066Z caller=checkpoint.go:372 msg="writing series" size="36 MB"
level=debug ts=2026-05-12T21:46:20.076264104Z caller=checkpoint.go:384 msg="flushing series" totalSize="37 MB" series=3
level=debug ts=2026-05-12T21:46:23.539818228Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-12T21:46:24.187201832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:25.386123459Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:46:25.386212453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:46:25.400773224Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:46:27.047258882Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-12T21:46:29.187750376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:30.552377242Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-12T21:46:34.059164287Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:46:34.188054247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:46:35.143774312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40350"
level=debug ts=2026-05-12T21:46:35.56368344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:46:37.565907812Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-12T21:46:39.18742024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:41.071821578Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
ts=2026-05-12T21:46:41.23799556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:46:42.296055667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:46:42.30520986Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T21:46:42.960836949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:46:42.969188504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:46:44.186969912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:46:44.18700312Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:46:44.187011359Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:46:44.187026555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:44.578862344Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-05-12T21:46:48.085059767Z caller=checkpoint.go:372 msg="writing series" size="742 kB"
level=debug ts=2026-05-12T21:46:48.085106762Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-12T21:46:49.188923557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:51.591528099Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T21:46:53.674251878Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:46:54.187203226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:55.097403238Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-12T21:46:58.604315089Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=warn ts=2026-05-12T21:46:59.187459969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:02.110619524Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:47:04.187743915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:05.617397163Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-12T21:47:08.351141088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:47:08.362991838Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:47:09.12382416Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-12T21:47:09.187279277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:47:11.239784471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:47:12.630862741Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=debug ts=2026-05-12T21:47:12.630883575Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-05-12T21:47:14.073082367Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:47:14.074115458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T21:47:14.187090013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:47:14.187134108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:47:14.187147219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:47:14.187159002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:47:14.202110829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:47:14.204015681Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:47:16.136884346Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-12T21:47:19.187963231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:47:19.194134201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:47:19.203083085Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:47:19.643343833Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-12T21:47:23.150327225Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:47:24.188201941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:47:24.653549077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45858"
level=debug ts=2026-05-12T21:47:24.79563885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:47:26.656234579Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=warn ts=2026-05-12T21:47:29.187566557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:30.162398059Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-12T21:47:32.866573258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:47:32.887181104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:47:33.672245991Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-12T21:47:34.188049079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:37.176198058Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:47:39.187342975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:40.682604077Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-12T21:47:41.241143695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:47:44.187832082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:47:44.187863311Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:47:44.187872048Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:47:44.187872615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:44.188954305Z caller=checkpoint.go:384 msg="flushing series" totalSize="764 kB" series=8
level=info ts=2026-05-12T21:47:44.193798769Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-12T21:47:44.196031606Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007857519s
level=warn ts=2026-05-12T21:47:49.187032023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:49.771056092Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:47:53.425593999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="306 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:47:53.438720992Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:47:54.187735123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:47:54.656771643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49232"
level=warn ts=2026-05-12T21:47:59.187736656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:48:03.200544642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:48:03.209178615Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:48:03.437598961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:48:04.187367181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:48:09.187485151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:48:11.242303314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:48:14.073851924Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:48:14.074338092Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=warn ts=2026-05-12T21:48:14.187033108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:48:14.187028486Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:48:14.187052299Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:48:14.187113951Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:48:14.187122662Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:48:14.187440474Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-12T21:48:14.187452134Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:48:14.187471557Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.333µs success=true
level=debug ts=2026-05-12T21:48:14.187647105Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-12T21:48:17.606287634Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T21:48:19.187655794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:21.023816111Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-12T21:48:23.506277722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:48:23.516001707Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:48:24.187038887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:24.441176333Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-12T21:48:24.660204207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57684"
level=debug ts=2026-05-12T21:48:25.390625094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:48:27.859674186Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-12T21:48:29.187308921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:31.277174854Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-12T21:48:34.187821447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:34.694850999Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-12T21:48:38.112027758Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-12T21:48:39.187606959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:48:41.243803409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:48:41.531174253Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T21:48:41.531198341Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=9
level=info ts=2026-05-12T21:48:44.186991816Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:48:44.187006603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:44.187023872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:48:44.187033743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:48:44.88768645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34762"
level=debug ts=2026-05-12T21:48:44.94857026Z caller=checkpoint.go:372 msg="writing series" size="630 kB"
level=info ts=2026-05-12T21:48:46.923189014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="306 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:48:46.931826876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:48:48.365063857Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T21:48:49.187869352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:51.134503153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:48:51.78342409Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-12T21:48:54.09305664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:48:54.187281466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:48:54.374222757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:48:54.383381891Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T21:48:54.664385886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:55186"
level=debug ts=2026-05-12T21:48:55.201464274Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-12T21:48:58.618153685Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-12T21:48:59.188091985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:02.035817686Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T21:49:04.187005871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:05.454280548Z caller=checkpoint.go:372 msg="writing series" size="405 kB"
level=debug ts=2026-05-12T21:49:05.454308999Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-05-12T21:49:08.871830444Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:49:09.187473754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:49:11.245677149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:49:12.289433969Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=info ts=2026-05-12T21:49:14.074116127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:49:14.074131882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:49:14.187201164Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:49:14.187207022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:14.18723226Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:49:14.187240778Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:49:15.707764462Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=info ts=2026-05-12T21:49:16.197012123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:49:16.20476046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:49:18.718787252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:49:19.124709111Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-12T21:49:19.187943294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:22.550691217Z caller=checkpoint.go:372 msg="writing series" size="7.2 MB"
level=debug ts=2026-05-12T21:49:22.550713288Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.8 MB" series=5
level=warn ts=2026-05-12T21:49:24.187954211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:25.960609675Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-12T21:49:29.187723543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:29.378590377Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T21:49:29.37861145Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=debug ts=2026-05-12T21:49:32.796131553Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:49:34.187150528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:36.214017446Z caller=checkpoint.go:372 msg="writing series" size="711 kB"
level=warn ts=2026-05-12T21:49:39.188105204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:39.631699939Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
ts=2026-05-12T21:49:41.248350214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:49:43.049216956Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=info ts=2026-05-12T21:49:43.480432224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:49:43.491252828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T21:49:44.187310536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:49:44.18735535Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:49:44.187357397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:49:44.187366303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:49:46.142755009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:49:46.466845827Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T21:49:49.188120981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:49.884175324Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-12T21:49:53.302029465Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-12T21:49:53.545579281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:49:54.030975142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:49:54.039394085Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T21:49:54.187310706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:56.719886704Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-12T21:49:56.786123462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:49:56.796687189Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:49:56.796715579Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:49:56.796750942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:49:56.805933229Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:49:59.188042058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:00.138237993Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-12T21:50:03.557980699Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-12T21:50:03.558009669Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=10
level=warn ts=2026-05-12T21:50:04.187229856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:06.972775957Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T21:50:09.187214283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:10.34914057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:50:10.390414297Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T21:50:11.250832648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:50:13.808638898Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T21:50:14.074110026Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:50:14.077202229Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:50:14.186965415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:50:14.186981507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:14.18700064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:50:14.187010247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:50:15.048016715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:50:15.055192413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:50:17.22653226Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-12T21:50:19.187007663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:20.644245181Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=debug ts=2026-05-12T21:50:24.061140431Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T21:50:24.187033881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:27.479435156Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-12T21:50:29.188035369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:50:30.6218985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:50:30.632406213Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:50:30.897315351Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T21:50:34.187016154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:34.31513924Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
level=debug ts=2026-05-12T21:50:34.31516203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-05-12T21:50:36.823786697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:50:37.732354495Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=warn ts=2026-05-12T21:50:39.187715808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:41.150158141Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-05-12T21:50:41.252753419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:50:41.848095018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:50:41.856360354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:50:44.186932413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:50:44.18697Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:50:44.186980706Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:50:44.18715906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:44.225441815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:50:44.571170633Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-12T21:50:44.894650448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56704"
level=debug ts=2026-05-12T21:50:47.985409786Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T21:50:47.985438039Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-12T21:50:49.187517395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:51.403109131Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-12T21:50:52.038910939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:50:52.047676706Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T21:50:54.187154067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:54.824914865Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T21:50:54.824935298Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=2
level=debug ts=2026-05-12T21:50:58.238789512Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:50:59.18927536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:01.65652964Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T21:51:04.18793642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:05.074301097Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=debug ts=2026-05-12T21:51:08.49257159Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-12T21:51:09.187779777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:51:11.254388989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:51:11.910229467Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T21:51:11.910255164Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-12T21:51:14.074019547Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:51:14.074123167Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:51:14.18733528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:51:14.18734607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:14.187379544Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:51:14.187400359Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:51:14.896494423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39544"
level=debug ts=2026-05-12T21:51:15.327661171Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T21:51:18.745865804Z caller=checkpoint.go:372 msg="writing series" size="925 kB"
level=warn ts=2026-05-12T21:51:19.187487636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:19.234697496Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:51:21.002449695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:51:21.004544777Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:51:22.203350499Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-12T21:51:22.203384132Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=3
level=warn ts=2026-05-12T21:51:24.187308004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:51:24.685661428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45968"
level=debug ts=2026-05-12T21:51:25.581095786Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-12T21:51:28.997914728Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-12T21:51:29.187653649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:51:29.303387123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:51:29.313081688Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:51:32.415808705Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-12T21:51:33.437627314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:51:33.444540737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:51:34.187517486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:35.83363263Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T21:51:39.187459161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:39.251736417Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-12T21:51:41.255615282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:51:41.662476337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:51:41.670816488Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:51:42.668925608Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=info ts=2026-05-12T21:51:44.187782084Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:51:44.187809007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:51:44.18781661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:51:44.187858291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:46.086900493Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-05-12T21:51:48.874812433Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:51:49.187921731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:49.505627461Z caller=checkpoint.go:372 msg="writing series" size="987 kB"
level=debug ts=2026-05-12T21:51:49.505647954Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=debug ts=2026-05-12T21:51:52.922709739Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:51:54.186993412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:56.340411993Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T21:51:59.188018152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:59.759028362Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T21:51:59.759047489Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-12T21:52:03.175499388Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:52:04.187621661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:06.593730996Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-12T21:52:09.187219665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:10.011645771Z caller=checkpoint.go:372 msg="writing series" size="922 kB"
level=debug ts=2026-05-12T21:52:10.011669722Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-05-12T21:52:11.257254664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:52:13.429214635Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-12T21:52:14.074153268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:52:14.074694341Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:52:14.1869503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:52:14.186977847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:14.18699653Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:52:14.187005994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T21:52:14.89858818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57944"
level=debug ts=2026-05-12T21:52:16.846724448Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-12T21:52:19.080509792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:52:19.187741425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:19.847840351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:52:20.263737163Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-12T21:52:22.911282722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:52:22.922942853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:52:23.282137389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:52:23.289693503Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:52:23.289714281Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:52:23.289749716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:52:23.29753084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:52:23.681618391Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:52:24.187994054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:27.100135078Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=warn ts=2026-05-12T21:52:29.187366503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:30.51739331Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-12T21:52:33.935443242Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-12T21:52:34.187018834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:52:35.183561989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50846"
level=debug ts=2026-05-12T21:52:37.352890568Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:52:39.187688006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:40.770434242Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-12T21:52:40.906880473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:52:40.919720738Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T21:52:41.258811861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:52:42.611310706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:52:42.620121817Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:52:44.187747952Z caller=checkpoint.go:384 msg="flushing series" totalSize="985 kB" series=9
level=info ts=2026-05-12T21:52:44.187754998Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:52:44.187779488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:52:44.187787943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:52:44.187809336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:52:44.192152078Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-12T21:52:44.194299149Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006688988s
level=warn ts=2026-05-12T21:52:49.187624706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:52:54.187019878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:54.226289379Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:52:54.226346712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:52:54.234626781Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T21:52:54.571571492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:52:54.695313084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:55380"
level=warn ts=2026-05-12T21:52:59.187885014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:53:04.187935643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:53:04.537093725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:53:04.545435795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:53:07.36798151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:53:07.376854395Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T21:53:08.347128153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:53:08.355703049Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:53:09.187669017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:53:11.260494689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:53:14.073481122Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:53:14.074546512Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:53:14.187221137Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:53:14.187250604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:53:14.187241991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:14.187269184Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:53:14.187277139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:53:14.187344757Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-12T21:53:14.187519587Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:53:14.187537925Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.396µs success=true
level=debug ts=2026-05-12T21:53:14.187756453Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-12T21:53:14.902615434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46916"
level=debug ts=2026-05-12T21:53:17.605822627Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T21:53:19.187798871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:21.024033784Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-12T21:53:21.155621842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:53:24.187129137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:24.441282673Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-12T21:53:24.697837926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:33980"
level=debug ts=2026-05-12T21:53:27.859588731Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T21:53:29.187826336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:31.277390881Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-12T21:53:32.297094348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:53:32.306754001Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T21:53:32.306789894Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:53:32.306824101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:53:32.316229268Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T21:53:34.18798893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:34.694606562Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=info ts=2026-05-12T21:53:37.026097635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:53:37.034121822Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T21:53:38.112527204Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T21:53:39.187811962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:53:41.261933235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:53:41.531455445Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T21:53:41.531480686Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=9
level=debug ts=2026-05-12T21:53:42.659016769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:53:43.057099438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:53:43.065377861Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T21:53:44.187537746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:53:44.187582306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:53:44.187593653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:53:44.18760017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:44.94790665Z caller=checkpoint.go:372 msg="writing series" size="648 kB"
level=info ts=2026-05-12T21:53:45.08467108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:53:45.092113957Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:53:48.365340989Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T21:53:48.915063242Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:53:49.188001941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:51.78365971Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-12T21:53:54.18700965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:53:54.700741696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36536"
level=debug ts=2026-05-12T21:53:55.200703786Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-12T21:53:58.618737541Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T21:53:59.188052269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:02.035965341Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T21:54:03.247538562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:54:03.257467806Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T21:54:04.187371447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:05.454219755Z caller=checkpoint.go:372 msg="writing series" size="604 kB"
level=debug ts=2026-05-12T21:54:05.454246889Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-05-12T21:54:08.340761434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:54:08.349533433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:54:08.871409337Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:54:09.187505862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:54:11.262957236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:54:12.289564941Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=info ts=2026-05-12T21:54:14.073866802Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:54:14.074591655Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:54:14.186936044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:54:14.186992017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:54:14.18701378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:54:14.187008158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:15.707681615Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=debug ts=2026-05-12T21:54:19.125172126Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-12T21:54:19.187360154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:21.463125581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:54:22.549455627Z caller=checkpoint.go:372 msg="writing series" size="7.8 MB"
level=debug ts=2026-05-12T21:54:22.549477138Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.6 MB" series=5
level=warn ts=2026-05-12T21:54:24.187369643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:54:24.704309091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60330"
level=debug ts=2026-05-12T21:54:25.960370585Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-12T21:54:29.187767847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:29.379265153Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T21:54:29.379300418Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=2
level=info ts=2026-05-12T21:54:29.97930851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:54:29.991877451Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:54:32.795832668Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-12T21:54:32.964809471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:54:34.187015579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:54:34.704689181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:54:34.713163357Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T21:54:35.20084916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52690"
level=debug ts=2026-05-12T21:54:36.213475675Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=warn ts=2026-05-12T21:54:39.18809779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:39.631309627Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-12T21:54:41.263841767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:54:43.048800827Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-12T21:54:44.187578004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:54:44.187583137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:44.187606675Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:54:44.187615284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:54:46.466883027Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T21:54:49.188030904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:49.884504758Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-12T21:54:51.846728661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:54:53.302152716Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T21:54:54.187003886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:56.720242189Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-12T21:54:59.180612241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=warn ts=2026-05-12T21:54:59.187663925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T21:54:59.189448984Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:55:00.137955971Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-12T21:55:00.490062377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:55:00.500198408Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:55:03.556559794Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T21:55:03.556581818Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=10
level=warn ts=2026-05-12T21:55:04.187310819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:06.973057744Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T21:55:09.187555742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:10.391222637Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-12T21:55:10.71303211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:55:10.715026184Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:55:11.264829852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:55:13.808386176Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-12T21:55:14.072393325Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:55:14.074491597Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:55:14.187320962Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:55:14.187350144Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:55:14.187357749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:55:14.187362642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:17.22608712Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-12T21:55:17.439713026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:55:17.447960294Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:55:19.188042763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:20.644030494Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=debug ts=2026-05-12T21:55:21.362349225Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:55:24.061944973Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T21:55:24.187841123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:27.479810376Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-12T21:55:29.187387786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:55:29.939948808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:55:29.95042218Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T21:55:30.896969365Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T21:55:34.187633926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:34.315615506Z caller=checkpoint.go:372 msg="writing series" size="594 kB"
level=debug ts=2026-05-12T21:55:34.315637505Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-12T21:55:37.733174528Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=warn ts=2026-05-12T21:55:39.18744504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:55:40.660078127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:55:40.673018389Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:55:41.150574023Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-05-12T21:55:41.26619348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:55:44.187198127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:55:44.187228819Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:55:44.187240794Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:55:44.187242548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:44.568038936Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-12T21:55:44.911000465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44018"
level=info ts=2026-05-12T21:55:45.983954122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:55:45.993176708Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:55:47.9865084Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T21:55:47.986530123Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-12T21:55:49.187913814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:51.40379906Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T21:55:54.187979604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:54.823903445Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T21:55:54.8239254Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=2
level=debug ts=2026-05-12T21:55:55.113798226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:55:55.979340803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:55:58.23943914Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:55:59.188031094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:01.656552038Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T21:56:04.187629983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:05.074666938Z caller=checkpoint.go:372 msg="writing series" size="503 kB"
ts=2026-05-12T21:56:05.206644004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48196"
level=info ts=2026-05-12T21:56:07.02555025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:56:07.034176719Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:56:07.034197071Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:56:07.034222291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:56:07.040751472Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:56:08.492393287Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-12T21:56:09.187292549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:56:11.26764862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:56:11.910198102Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T21:56:11.910220433Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-12T21:56:14.072939944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:56:14.073990041Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:56:14.187479912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:56:14.18748801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:14.187508384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:56:14.18751637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:56:15.327130062Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:56:15.777447496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:56:15.785137013Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T21:56:18.745983008Z caller=checkpoint.go:372 msg="writing series" size="861 kB"
level=warn ts=2026-05-12T21:56:19.187797712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:56:21.208564538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:56:21.216812458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:56:22.19863551Z caller=checkpoint.go:372 msg="writing series" size="47 MB"
level=debug ts=2026-05-12T21:56:22.198660736Z caller=checkpoint.go:384 msg="flushing series" totalSize="48 MB" series=3
level=debug ts=2026-05-12T21:56:23.789761593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:56:24.187254727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:25.581236847Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-12T21:56:28.150593158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:56:28.160753592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:56:28.998055457Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:56:29.187368698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:32.417481655Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T21:56:32.417507127Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-12T21:56:34.187530176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:56:35.209306348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51930"
level=info ts=2026-05-12T21:56:35.779720385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:56:35.78771015Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:56:35.833779198Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:56:39.187628125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:39.251876548Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-12T21:56:41.270127908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:56:42.669494713Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-12T21:56:44.186956751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:56:44.186988352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:56:44.186996897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:56:44.187043323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:45.212370971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:56:46.087556754Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-12T21:56:49.187653844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:49.505734249Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T21:56:49.505755158Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-12T21:56:50.106631725Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:56:52.412196531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:56:52.420905366Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:56:52.922203888Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:56:54.187810965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:56:54.720288053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53568"
level=debug ts=2026-05-12T21:56:56.340189665Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-12T21:56:58.460099078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:56:58.470016687Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T21:56:59.18763959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:59.758520284Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T21:56:59.758539084Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-12T21:57:03.175546931Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:57:04.188098076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:57:05.211177922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:49426"
level=debug ts=2026-05-12T21:57:06.592968765Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-12T21:57:09.187399799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:10.011769369Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T21:57:10.01179284Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T21:57:11.271903804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:57:13.428983192Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T21:57:14.073799207Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:57:14.073834658Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:57:14.187716478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:57:14.187748146Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:57:14.187757123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:57:14.187744186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:16.230947129Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:57:16.846639753Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-12T21:57:19.187308872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:20.264376281Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-12T21:57:23.681967371Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:57:24.18718564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:57:24.724092026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40418"
level=info ts=2026-05-12T21:57:24.786210201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:57:24.798452092Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:57:27.100252571Z caller=checkpoint.go:372 msg="writing series" size="433 kB"
level=debug ts=2026-05-12T21:57:27.100276812Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-12T21:57:27.53819057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:57:27.546681493Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T21:57:29.187859499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:30.517188294Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-12T21:57:33.934691438Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-12T21:57:34.187523723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:57:35.212391472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54226"
level=debug ts=2026-05-12T21:57:37.353196179Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:57:39.187829669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:39.229580532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:57:40.770303173Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
ts=2026-05-12T21:57:41.273160093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T21:57:44.1877082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:57:44.187738848Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:57:44.187747565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:57:44.187759225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:44.187753083Z caller=checkpoint.go:384 msg="flushing series" totalSize="260 kB" series=4
level=info ts=2026-05-12T21:57:44.195798671Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-12T21:57:44.198891848Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011175034s
ts=2026-05-12T21:57:44.918976935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47502"
level=info ts=2026-05-12T21:57:45.224935354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:57:45.232020751Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:57:46.390408569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:57:49.187678882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:57:54.18794159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:57:58.071315154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:57:58.083437836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T21:57:59.187941443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:58:04.187184991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:58:07.888379095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:58:07.898131112Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:58:09.187712991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:58:11.274369033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:58:14.072479452Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:58:14.074591371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T21:58:14.187590961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:58:14.187606415Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:58:14.187630718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:58:14.18764964Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:58:14.187657334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:58:14.187762071Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-12T21:58:14.187770636Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:58:14.187789629Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.688µs success=true
level=debug ts=2026-05-12T21:58:14.187971209Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-12T21:58:17.606461445Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T21:58:19.188052343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:20.410190299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:58:21.023946485Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-12T21:58:24.187261089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:24.442142365Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-12T21:58:24.730407336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54686"
level=info ts=2026-05-12T21:58:25.948391691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:58:25.958171403Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:58:27.859807463Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T21:58:29.187741437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:31.277562059Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-12T21:58:34.187591801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:34.69479543Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=info ts=2026-05-12T21:58:35.706848281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:58:35.713520704Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:58:37.718874817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:58:38.112219223Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T21:58:39.187670935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:58:41.276341792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:58:41.532705199Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T21:58:41.532732572Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=9
level=info ts=2026-05-12T21:58:44.187267229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:58:44.18730384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:58:44.187314141Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:58:44.187335488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:44.948322984Z caller=checkpoint.go:372 msg="writing series" size="741 kB"
level=debug ts=2026-05-12T21:58:48.365534201Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T21:58:49.188015721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:51.783085479Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-12T21:58:52.702686805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:58:52.711861392Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T21:58:54.187745052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:55.201064916Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-12T21:58:55.995009114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:58:58.619439915Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-12T21:58:58.619460132Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-12T21:58:59.18810531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:59:01.361648652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:01.368992641Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:01.369016494Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:59:01.369046272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:01.375711701Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:59:02.036412345Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T21:59:04.187855785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:05.454314279Z caller=checkpoint.go:372 msg="writing series" size="363 kB"
level=debug ts=2026-05-12T21:59:08.872014768Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:59:09.187387215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:59:11.278843618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T21:59:11.633312229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:11.643146126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:59:12.290224041Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=debug ts=2026-05-12T21:59:14.072801204Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T21:59:14.073867879Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:59:14.187359872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:59:14.187384899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:59:14.187393121Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:59:14.187386789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:15.707797247Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-05-12T21:59:15.707818963Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-12T21:59:19.12518431Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-12T21:59:19.187431396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:20.37096857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:59:20.372860844Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:59:21.23718413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T21:59:21.2439679Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T21:59:22.552040782Z caller=checkpoint.go:372 msg="writing series" size="8.6 MB"
level=debug ts=2026-05-12T21:59:22.552062453Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.7 MB" series=2
level=info ts=2026-05-12T21:59:23.712127454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:59:23.719096101Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:59:24.020000851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:59:24.187327822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:25.960867662Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-12T21:59:29.187722757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:29.379346442Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T21:59:29.379366673Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=2
level=debug ts=2026-05-12T21:59:32.796475006Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:59:34.18753314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:59:35.220409483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33936"
level=debug ts=2026-05-12T21:59:36.213967322Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=warn ts=2026-05-12T21:59:39.187983028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:39.631738234Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
ts=2026-05-12T21:59:41.280343405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T21:59:43.049425493Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-12T21:59:43.865004464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:43.876229471Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T21:59:44.187955255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:59:44.187987508Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:59:44.187997175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:59:44.188031964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:59:45.381823149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:59:45.39137017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:59:45.391391593Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:59:45.391417056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:59:45.399185856Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T21:59:46.467221411Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T21:59:49.187386513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:49.885090102Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-12T21:59:53.302332575Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T21:59:54.187846141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:55.443065359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:59:56.719993359Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-12T21:59:58.250536593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T21:59:58.260225755Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T21:59:59.187313087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:00.138252101Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=debug ts=2026-05-12T22:00:03.557282277Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:00:03.557304475Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=10
level=warn ts=2026-05-12T22:00:04.187582401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:06.973476195Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-12T22:00:08.386631893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:00:08.394514633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:00:09.187431663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:09.702475734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:00:10.391459882Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T22:00:11.281985519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:00:13.490660748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:00:13.498709595Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:00:13.498727948Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:00:13.498752946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:00:13.506336901Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:00:13.809132424Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-12T22:00:14.072655807Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:00:14.074775321Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:00:14.076047897Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:00:14.076111964Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976247 n=1
level=debug ts=2026-05-12T22:00:14.076153196Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976247 n=1
level=info ts=2026-05-12T22:00:14.187739561Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:00:14.187767387Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:00:14.18775664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:00:14.187774977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:00:17.226035538Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-12T22:00:19.187282979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:20.643874446Z caller=checkpoint.go:372 msg="writing series" size="390 kB"
level=debug ts=2026-05-12T22:00:24.061583377Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T22:00:24.187876166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:00:26.933723077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:00:26.946016492Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:00:27.479596063Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-12T22:00:27.586898125Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:00:29.188007555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:30.897229548Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-12T22:00:32.568992582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:00:32.577739142Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:00:34.186996947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:34.314910142Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=debug ts=2026-05-12T22:00:34.314930335Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-05-12T22:00:37.73260474Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=warn ts=2026-05-12T22:00:39.187150212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:41.150247978Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
ts=2026-05-12T22:00:41.284083328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:00:43.504044882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:00:43.514114516Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:00:44.187508705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:00:44.18753877Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:00:44.187547061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:00:44.187568287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:44.567877619Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-12T22:00:47.986566019Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:00:47.986588118Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-12T22:00:49.187481937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:51.40403979Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=debug ts=2026-05-12T22:00:52.565952316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:00:54.187541876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:54.824163517Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:00:54.824184295Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=2
level=info ts=2026-05-12T22:00:55.220783213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:00:55.228802033Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:00:56.113685967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:00:58.238707197Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T22:00:59.187587574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:01.656513468Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:01:04.187609805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:05.074785223Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=info ts=2026-05-12T22:01:07.822593705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:01:07.832102766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:01:08.49269732Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-12T22:01:09.187825789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:01:11.287463682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:01:11.909942233Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:01:11.909960566Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-05-12T22:01:14.072999047Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:01:14.074058325Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:01:14.187873155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:01:14.187873179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:01:14.187896443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:01:14.18790397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:01:15.328134926Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T22:01:18.745860367Z caller=checkpoint.go:372 msg="writing series" size="898 kB"
level=warn ts=2026-05-12T22:01:19.187170884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:01:19.580059359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:01:19.588677743Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:01:22.201100923Z caller=checkpoint.go:372 msg="writing series" size="47 MB"
level=debug ts=2026-05-12T22:01:22.201125264Z caller=checkpoint.go:384 msg="flushing series" totalSize="48 MB" series=3
level=debug ts=2026-05-12T22:01:23.747996767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:01:24.187236155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:25.580640495Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-12T22:01:28.998644173Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T22:01:29.187628197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:32.417479858Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:01:32.417503533Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-12T22:01:34.187700629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:01:35.177839079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:01:35.187454678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:01:35.834112229Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:01:36.508349354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:01:36.515977019Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:01:38.890370172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:01:39.188044174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:39.252219777Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
ts=2026-05-12T22:01:41.289881431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:01:42.669899319Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-12T22:01:44.187730648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:01:44.187743179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:44.187761665Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:01:44.187769663Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T22:01:44.934881775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44546"
level=debug ts=2026-05-12T22:01:46.086973935Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-12T22:01:49.187453612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:49.50663109Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:01:49.506652413Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-12T22:01:52.923214594Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:01:54.187889407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:56.340870705Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-12T22:01:58.617353467Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:01:59.18722059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:59.758951959Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:01:59.758971145Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-12T22:02:03.175716896Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T22:02:04.187028246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:02:05.222586728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:05.234662133Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:02:06.594182897Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-12T22:02:09.187530611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:10.011750147Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:02:10.011771514Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:02:11.290975508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:02:13.429567078Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T22:02:14.073416404Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:02:14.074476731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:02:14.187338207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:02:14.187366557Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:02:14.187378142Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:02:14.187418446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:02:14.936394478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38936"
level=info ts=2026-05-12T22:02:16.012052475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:02:16.021996249Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:02:16.846295892Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-12T22:02:19.188038033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:20.26494236Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=info ts=2026-05-12T22:02:21.067115653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:02:21.075759761Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:02:22.140644553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:02:22.148350649Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:02:23.682574124Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T22:02:24.187559997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:24.517012294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:02:27.100042409Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=warn ts=2026-05-12T22:02:29.187976363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:30.518117277Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-12T22:02:30.518137656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-12T22:02:33.241618229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:02:33.935598432Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T22:02:34.187127874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:37.353406494Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T22:02:39.188031822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:40.770760842Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-05-12T22:02:41.292743186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:02:44.187296033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:02:44.187293447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:44.187334785Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:02:44.18734287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:02:44.188385674Z caller=checkpoint.go:384 msg="flushing series" totalSize="212 kB" series=3
level=info ts=2026-05-12T22:02:44.196388742Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-12T22:02:44.200093008Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012159981s
ts=2026-05-12T22:02:44.93813558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38030"
level=warn ts=2026-05-12T22:02:49.186993045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:02:54.187920787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:02:54.288706171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:54.297191488Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:54.297216385Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:02:54.297242466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:54.303988363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:02:54.757534084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60082"
level=warn ts=2026-05-12T22:02:59.187866343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:01.63897293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:03:03.328487983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:03:03.336043908Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:03:04.18700112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:03:05.235547342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56886"
level=info ts=2026-05-12T22:03:08.431846064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:03:08.439293669Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:03:09.187182405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:03:11.294112555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:03:12.28155274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:03:12.289376172Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:03:14.072488955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:03:14.074581824Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:03:14.187300005Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:03:14.187370917Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:03:14.187391572Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:03:14.187402612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:03:14.187421811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:03:14.187458951Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-12T22:03:14.187468247Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:03:14.187489294Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.164µs success=true
level=debug ts=2026-05-12T22:03:14.187701Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-12T22:03:14.938982216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57426"
level=info ts=2026-05-12T22:03:17.132979302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:03:17.140460424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:03:17.60590168Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:03:19.187770312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:21.023965653Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-12T22:03:23.615716205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:03:23.61724386Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:03:24.187062836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:24.440929375Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-12T22:03:27.580729823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:03:27.859235335Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:03:28.647536054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:03:28.655713314Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:03:29.187949981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:31.277158072Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-12T22:03:34.187120372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:34.694118295Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=debug ts=2026-05-12T22:03:38.112940403Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:03:39.187994197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:03:41.295544214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:03:41.53101059Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-12T22:03:41.53103308Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=9
level=info ts=2026-05-12T22:03:44.186966821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:03:44.186999183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:03:44.187007212Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:03:44.187033298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:44.948643686Z caller=checkpoint.go:372 msg="writing series" size="754 kB"
level=debug ts=2026-05-12T22:03:48.365610234Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:03:49.187243549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:03:49.581780561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:03:49.590820363Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:03:51.783251684Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-12T22:03:54.013551685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:03:54.022922664Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:03:54.187155963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:54.747062678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:03:55.201061685Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-12T22:03:58.618408465Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-12T22:03:58.618431986Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-12T22:03:59.187311283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:59.753857328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:04:00.04598077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:04:00.053038065Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:04:02.036055456Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T22:04:04.187192642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:05.454225205Z caller=checkpoint.go:372 msg="writing series" size="384 kB"
level=debug ts=2026-05-12T22:04:08.871947344Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T22:04:09.18803382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:04:11.297235965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:04:12.2897398Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=debug ts=2026-05-12T22:04:14.073618272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:04:14.074672641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:04:14.187597355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:04:14.187642461Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:04:14.18765172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:04:14.187663004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:15.707696407Z caller=checkpoint.go:372 msg="writing series" size="665 kB"
level=debug ts=2026-05-12T22:04:15.707720067Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-12T22:04:18.416866301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:04:18.424508791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:04:19.124979739Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-12T22:04:19.187426026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:22.551148429Z caller=checkpoint.go:372 msg="writing series" size="9.3 MB"
level=debug ts=2026-05-12T22:04:22.551180873Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.4 MB" series=2
level=debug ts=2026-05-12T22:04:23.439259474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:04:23.439297588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:04:23.445275937Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:04:23.740127097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:04:23.744529038Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:04:24.040291143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:04:24.047628962Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:04:24.187059594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:04:24.767271183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:38584"
level=debug ts=2026-05-12T22:04:25.960244153Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-12T22:04:29.187886978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:29.3794392Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:04:29.379460931Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=debug ts=2026-05-12T22:04:32.795920544Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:04:34.187747973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:04:35.242022296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:38212"
level=debug ts=2026-05-12T22:04:36.213834302Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=info ts=2026-05-12T22:04:37.87611273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:04:37.88251711Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:04:38.18352225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:04:39.187342257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:04:39.625580287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:04:39.630989197Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=error ts=2026-05-12T22:04:39.635201922Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:04:41.045907274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:04:41.053360358Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T22:04:41.299409649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:04:42.057818149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:04:42.062795563Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:04:43.049416511Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-12T22:04:43.099941663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:04:43.104140227Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:04:44.187161083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:04:44.187175465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:04:44.187202516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:04:44.187211023Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:04:46.467165902Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-12T22:04:48.805685197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:04:48.812838237Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:04:49.187883771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:49.884974231Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-12T22:04:53.302148816Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:04:54.187647257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:04:54.770664536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53032"
level=debug ts=2026-05-12T22:04:56.720295848Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-12T22:04:58.97496152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:04:59.18775148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:00.137547994Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-12T22:05:02.693789716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:05:02.706038035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:05:03.556851986Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-12T22:05:03.556874376Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=10
level=warn ts=2026-05-12T22:05:04.187978199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:05:05.24285657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:05:05.247738761Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:05:06.972674197Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-12T22:05:07.351803338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:05:07.35656374Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:05:09.187205322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:10.390759129Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T22:05:11.301617353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:05:13.808140221Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-12T22:05:14.073127695Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:05:14.074174691Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:05:14.186973604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:05:14.187003322Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:05:14.187012555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:05:14.187029364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:17.225748704Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-12T22:05:18.124796095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:05:18.131634402Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:05:19.187987187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:20.644207394Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=debug ts=2026-05-12T22:05:24.061605906Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T22:05:24.187539193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:05:24.77369591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36904"
level=debug ts=2026-05-12T22:05:27.479754404Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=warn ts=2026-05-12T22:05:29.187214402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:05:29.196340284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:05:29.204744359Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:05:30.323299811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:05:30.896940748Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T22:05:34.187424046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:34.31548607Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-12T22:05:34.315506494Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-12T22:05:34.374398849Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:05:37.594199941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:05:37.601667086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:05:37.732410179Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=warn ts=2026-05-12T22:05:39.187901129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:41.150429521Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
ts=2026-05-12T22:05:41.303339275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:05:44.187595483Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:05:44.18759653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:44.187622658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:05:44.187631378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:05:44.568124829Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-12T22:05:44.705726748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:05:44.71309678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:05:44.713114968Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:05:44.713150017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:05:44.720043667Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:05:47.986358282Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:05:47.986380007Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-12T22:05:48.701127709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:05:48.708615828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:05:49.187604081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:05:50.76448448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:05:50.769236972Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:05:51.403231052Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=warn ts=2026-05-12T22:05:54.187023934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:05:54.770567617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:05:54.775403805Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:05:54.822493231Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:05:54.822511112Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=debug ts=2026-05-12T22:05:58.239200829Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T22:05:59.187990396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:59.80859108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:06:01.657255078Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:06:04.017291951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:04.02824507Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:06:04.187606392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:05.074018399Z caller=checkpoint.go:372 msg="writing series" size="474 kB"
level=debug ts=2026-05-12T22:06:08.492187446Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-12T22:06:09.188036524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:06:11.305758502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:06:11.910323398Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:06:11.910345867Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-12T22:06:14.072208305Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:06:14.074330664Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:06:14.186997038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:06:14.186996451Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:06:14.187028859Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:06:14.187038832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:06:14.356109062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:06:14.36149803Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:06:15.327224519Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T22:06:18.745040806Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-05-12T22:06:19.187991813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:22.203699272Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-12T22:06:22.203727808Z caller=checkpoint.go:384 msg="flushing series" totalSize="49 MB" series=3
level=warn ts=2026-05-12T22:06:24.187317749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:24.877544125Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:06:25.580460311Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-12T22:06:25.794357962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:06:28.998040989Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-12T22:06:29.04403586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:06:29.053920852Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:06:29.053941062Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:06:29.053964401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:06:29.061738196Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:06:29.187692297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:06:31.99288266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:06:31.99716104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:06:32.41735984Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:06:32.417380446Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-05-12T22:06:34.187836909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:06:34.781342551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:06:34.792233651Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:06:34.792254124Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:06:34.792278888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:06:34.799860082Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:06:35.487562772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:06:35.495817413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:06:35.834213416Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T22:06:39.187861615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:39.2520888Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-12T22:06:39.637862436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:06:39.64624839Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:06:40.924706485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:40.93479128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:06:41.308338753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:06:42.66969716Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-12T22:06:44.187580334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:06:44.187606907Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:06:44.187614901Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:06:44.187620546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:06:44.951214335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38774"
level=debug ts=2026-05-12T22:06:46.086934445Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-12T22:06:47.743348188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:06:47.752735552Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:06:49.187956204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:49.506282026Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:06:49.506301796Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-12T22:06:52.491917527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:06:52.496139354Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:06:52.92219878Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:06:54.187923035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:06:54.784019679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56558"
level=debug ts=2026-05-12T22:06:56.339970999Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-12T22:06:59.18750818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:59.524418845Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:06:59.758095309Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:06:59.758117678Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-12T22:07:03.17547855Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T22:07:04.186980748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:07:05.253505235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:38148"
level=info ts=2026-05-12T22:07:05.613024503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:07:05.623038447Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:07:06.593463663Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=warn ts=2026-05-12T22:07:09.187752831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:10.012099117Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:07:10.012125916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-12T22:07:11.310327156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:07:12.842914772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:07:12.85101851Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:07:13.429202415Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T22:07:14.072490574Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:07:14.074625501Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:07:14.187839833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:07:14.187870678Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:07:14.187853634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:07:14.187878044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:07:16.846060915Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-12T22:07:18.302799127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:07:18.310420256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:07:19.187706544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:20.264377395Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-12T22:07:23.68355222Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T22:07:24.187815089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:07:25.4515337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:07:25.459347688Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:07:26.080984116Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:07:27.099590152Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
level=debug ts=2026-05-12T22:07:27.099611881Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-12T22:07:27.295857445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:07:27.303592823Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:07:27.359427059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:07:27.360923992Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:07:28.828985216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:07:28.833269269Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:07:29.188048869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:30.517098646Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-12T22:07:33.935232903Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T22:07:34.187419331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:37.352988669Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T22:07:39.18775531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:40.770910451Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
ts=2026-05-12T22:07:41.313766917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:07:44.187034652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:07:44.187062052Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:07:44.187095151Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:07:44.187102169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:44.188173026Z caller=checkpoint.go:384 msg="flushing series" totalSize="287 kB" series=4
level=info ts=2026-05-12T22:07:44.194813221Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-12T22:07:44.1975252Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009863678s
level=info ts=2026-05-12T22:07:44.630097784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:07:44.635188262Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:07:48.392984649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:07:48.40136693Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:07:49.188092294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:07:54.18700463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:54.949199759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:07:59.187336296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:08:04.187779796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:08:09.18798571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:10.279840033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:08:10.606668815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:08:10.61494846Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T22:08:11.316319011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:08:12.072194737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:08:12.079477799Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:08:14.072633704Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:08:14.074755954Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:08:14.187600075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:08:14.187628438Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:08:14.187636207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:08:14.187623066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:08:14.187632002Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:08:14.187774337Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-12T22:08:14.187782424Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:08:14.187803077Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.837µs success=true
level=debug ts=2026-05-12T22:08:14.187979365Z caller=checkpoint.go:372 msg="writing series" size="639 B"
ts=2026-05-12T22:08:14.958445775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38574"
level=info ts=2026-05-12T22:08:15.759428656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:08:15.767198073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:08:17.606502936Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:08:19.187757888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:08:19.368031427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:08:19.372109707Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:08:21.024351401Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-12T22:08:24.187665657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:24.441394915Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-12T22:08:24.772645804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:08:24.780200577Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:08:24.793309946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49414"
level=debug ts=2026-05-12T22:08:26.807191455Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:08:27.85963402Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T22:08:29.187955315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:31.276825313Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-12T22:08:32.972653191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:08:32.97873846Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:08:34.187290777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:34.694746019Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-12T22:08:34.72180635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:08:34.72771036Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:08:38.112769669Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:08:39.187251837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:08:41.317604634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:08:41.531834554Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-12T22:08:41.531863268Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=9
level=info ts=2026-05-12T22:08:44.187594542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:08:44.187627506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:08:44.187636792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:08:44.187666117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:44.948669344Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
ts=2026-05-12T22:08:44.959378571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55150"
level=debug ts=2026-05-12T22:08:48.365687785Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:08:49.187812319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:51.783955085Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-12T22:08:54.054141902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:08:54.061053471Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:08:54.1877854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:55.200749719Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-12T22:08:55.577773564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:08:58.619242089Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T22:08:59.187577859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:59.961214018Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:09:02.037049276Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T22:09:02.266982186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:09:02.274489416Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:09:04.187065498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:05.454732244Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=debug ts=2026-05-12T22:09:05.454771504Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-12T22:09:06.923724296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:06.932358392Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:06.932383061Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:09:06.932405367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:06.939863614Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:09:07.547518913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:09:07.552250256Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:09:08.872165913Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T22:09:09.188014676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:09:11.319894433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:09:12.289302886Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=debug ts=2026-05-12T22:09:14.072444159Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:09:14.074599933Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:09:14.187535951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:09:14.187562711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:09:14.187570949Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:09:14.187567288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:15.708265517Z caller=checkpoint.go:372 msg="writing series" size="800 kB"
level=info ts=2026-05-12T22:09:16.498314677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:09:16.50787109Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:09:17.20962706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:09:17.215277935Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:09:18.651044404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:18.658688861Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:09:19.125530021Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-12T22:09:19.12555131Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-12T22:09:19.187720448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:22.551798261Z caller=checkpoint.go:372 msg="writing series" size="9.7 MB"
level=debug ts=2026-05-12T22:09:22.55181955Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=1
level=warn ts=2026-05-12T22:09:24.187666492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:25.961099666Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-12T22:09:28.508115987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:09:28.513295182Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:09:29.18799329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:29.380679407Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:09:29.38070599Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=debug ts=2026-05-12T22:09:31.235355051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:09:32.795913095Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:09:34.187640583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:36.214236065Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=warn ts=2026-05-12T22:09:39.188004912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:39.631278417Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=debug ts=2026-05-12T22:09:40.035346022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:09:41.322118923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:09:43.049438798Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-12T22:09:44.18723437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:09:44.187241735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:44.187277313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:09:44.18728561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:09:45.568385738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:09:45.575696846Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:09:45.740124355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:09:45.747124882Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:09:46.467350995Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-12T22:09:47.901230199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:47.909322744Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:09:49.187530916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:49.884966859Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-12T22:09:50.977850305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:09:50.982637097Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:09:53.302065722Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:09:54.187192324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:56.720148967Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T22:09:59.187648396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:59.653674781Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:10:00.138497503Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-12T22:10:03.55767407Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-12T22:10:03.557697928Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=10
level=warn ts=2026-05-12T22:10:04.187923201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:06.972936514Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T22:10:09.187668727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:10.391119493Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T22:10:11.323941891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:10:12.943906627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:10:12.951601301Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:10:13.80931709Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-12T22:10:14.07272806Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:10:14.074840552Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:10:14.187702061Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:10:14.187722225Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:10:14.18773189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:10:14.187758085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:15.239757072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:10:15.508654556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:10:15.512822403Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:10:17.156403209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:10:17.161209949Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:10:17.226339108Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-12T22:10:19.187441878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:20.644110658Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=debug ts=2026-05-12T22:10:24.062148117Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T22:10:24.187045555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:24.773711915Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:10:27.479470687Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-12T22:10:29.181755761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:10:29.187734583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:10:29.193234864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:10:30.896898336Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-12T22:10:31.965828535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:10:31.970156912Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:10:31.970178316Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:10:31.97019942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:10:31.973996448Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:10:33.661612531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:10:33.669063999Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:10:34.187355364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:34.315350426Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=debug ts=2026-05-12T22:10:34.315368715Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-05-12T22:10:37.733187972Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=warn ts=2026-05-12T22:10:39.187789598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:41.150770337Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:10:41.325446492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:10:41.790368898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:10:41.799104143Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:10:44.187548729Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:10:44.187581947Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:10:44.18759036Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:10:44.187605918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:44.568216724Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-12T22:10:47.490141636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:10:47.495598915Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:10:47.986140427Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:10:47.986161797Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-12T22:10:49.187547733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:51.403642394Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-12T22:10:52.4354896Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:10:54.187708077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:54.823308937Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:10:54.823327253Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=2
level=debug ts=2026-05-12T22:10:58.239085599Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:10:58.554081054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:10:58.561033215Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:10:58.561051397Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:10:58.561091841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:10:58.565834562Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:10:59.187468949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:59.209727955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:10:59.211318391Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:11:01.234607542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:11:01.241694616Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:11:01.65698108Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:11:04.187985013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:05.074303707Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=debug ts=2026-05-12T22:11:08.492412136Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-12T22:11:09.187666852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:11:11.326555108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:11:11.910349976Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:11:11.910460976Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-12T22:11:13.986206204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:11:13.99286928Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:11:13.992888019Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:11:13.992911911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:11:13.997301429Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:11:14.073021788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:11:14.074091082Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:11:14.186975354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:11:14.186980914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:14.187005745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:11:14.187013255Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T22:11:14.968861255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55912"
level=debug ts=2026-05-12T22:11:15.32743829Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:11:16.238924979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:11:16.246227082Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:11:17.690316348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:11:17.695923449Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:11:17.706988407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:11:17.711205002Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:11:18.745953219Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:11:18.745974422Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-12T22:11:19.187963055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:22.205557347Z caller=checkpoint.go:372 msg="writing series" size="49 MB"
level=debug ts=2026-05-12T22:11:22.205583939Z caller=checkpoint.go:384 msg="flushing series" totalSize="49 MB" series=1
level=debug ts=2026-05-12T22:11:23.815946002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:11:24.187100883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:25.580835682Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-12T22:11:28.423546706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:11:28.430425005Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:11:28.999012697Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-12T22:11:29.188017743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:11:30.609386021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:11:30.62243715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:11:32.417661249Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:11:32.417681665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-05-12T22:11:34.187030204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:11:35.27629411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58018"
level=debug ts=2026-05-12T22:11:35.833952067Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-12T22:11:36.221038568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:11:39.187453715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:39.251674143Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-12T22:11:41.033191256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:11:41.044027033Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T22:11:41.329242787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:11:42.670084236Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-12T22:11:42.960702635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:11:42.966550708Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:11:44.188143484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:11:44.188172752Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:11:44.188180983Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:11:44.188198072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:46.08727033Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-12T22:11:49.187700117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:49.507039969Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:11:49.507062543Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-12T22:11:52.923146331Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:11:54.187947197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:11:54.612682343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:11:54.624292846Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:11:56.340632718Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-12T22:11:57.851666934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:11:59.187523793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:11:59.588726469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:11:59.593437194Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:11:59.759351972Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:11:59.759369961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:12:00.200675159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:12:00.205876104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:12:03.175766485Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T22:12:04.18702184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:06.594213694Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=warn ts=2026-05-12T22:12:09.187873795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:10.011644116Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:12:10.011665715Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-05-12T22:12:11.331601763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:12:13.429797908Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=info ts=2026-05-12T22:12:14.073881219Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:12:14.073882749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:12:14.187831338Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:12:14.187860607Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:12:14.187867966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:12:14.187867323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:15.576799985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:12:15.585693305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:12:16.846455238Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-12T22:12:19.187697274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:20.265208949Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-12T22:12:21.737197414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:21.744217284Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:21.74424904Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:12:21.744282156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:21.751539133Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:12:23.361561978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:12:23.682512648Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-12T22:12:24.075356692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:12:24.187801389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:24.566517868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:12:24.574327168Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:12:24.616404514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:12:24.621209655Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
ts=2026-05-12T22:12:24.819162046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57380"
level=debug ts=2026-05-12T22:12:27.100205922Z caller=checkpoint.go:372 msg="writing series" size="408 kB"
level=debug ts=2026-05-12T22:12:27.100226396Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-12T22:12:29.187429805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:30.51754242Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:12:33.935225244Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-12T22:12:34.187971761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:34.794465039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:12:34.804971261Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:12:37.352638283Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T22:12:39.187994199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:39.9139575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:39.921587054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:12:40.770427233Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
ts=2026-05-12T22:12:41.334114996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:12:44.187207528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:12:44.187239073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:12:44.187247329Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:12:44.187243682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:44.188325006Z caller=checkpoint.go:384 msg="flushing series" totalSize="307 kB" series=4
level=info ts=2026-05-12T22:12:44.193606449Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-12T22:12:44.195585448Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007647366s
level=warn ts=2026-05-12T22:12:49.187292604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:12:54.187342638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:12:54.822340266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49776"
level=debug ts=2026-05-12T22:12:57.909403078Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:12:58.074987246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:12:58.080823175Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:12:59.187970726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:00.119640706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:13:00.124483172Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:13:03.145125786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:13:03.152536759Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:13:04.187650917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:13:05.280523568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52504"
level=warn ts=2026-05-12T22:13:09.187813719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:10.620937433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:13:10.62797896Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
ts=2026-05-12T22:13:11.337164638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:13:13.654233366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:13:14.072415338Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:13:14.074536279Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:13:14.18756825Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:13:14.18761916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:14.187637121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:13:14.187671696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:13:14.187684058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:13:14.18771266Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-12T22:13:14.187721047Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:13:14.187739625Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.366µs success=true
level=debug ts=2026-05-12T22:13:14.18791183Z caller=checkpoint.go:372 msg="writing series" size="639 B"
ts=2026-05-12T22:13:14.9758458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51888"
level=info ts=2026-05-12T22:13:15.742197511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:13:15.75471132Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:13:15.754735793Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:13:15.754760763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:13:15.762607747Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:13:17.606597398Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:13:19.187720105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:19.476868299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:13:19.485628914Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:13:21.023942345Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-12T22:13:23.222380855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:13:24.188100079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:24.441151951Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-12T22:13:24.82585357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:43156"
level=info ts=2026-05-12T22:13:26.708896656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:13:26.717162791Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:13:27.859608515Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T22:13:29.186989759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:31.276965327Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-12T22:13:32.595539093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:32.603741984Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:32.603769178Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:13:32.603792931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:32.611046933Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:13:34.187656982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:34.695140749Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T22:13:38.113211084Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:13:39.187642845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:13:41.339303688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:13:41.532088097Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-12T22:13:41.532114344Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=9
level=info ts=2026-05-12T22:13:44.187133819Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:13:44.187149518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:44.187170383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:13:44.187181201Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:13:44.948568137Z caller=checkpoint.go:372 msg="writing series" size="718 kB"
ts=2026-05-12T22:13:44.977049329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55390"
level=info ts=2026-05-12T22:13:46.08671542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:46.097989058Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:13:48.366131732Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:13:49.187919239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:51.530434954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:13:51.538142818Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:13:51.785387558Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-12T22:13:51.828367848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:13:51.833613088Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:13:52.456394827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:13:52.461197766Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:13:53.406945778Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:13:54.187789322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:54.722420848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:13:55.201658498Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-12T22:13:55.20168893Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-12T22:13:58.618869401Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-12T22:13:59.187274077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:59.493503471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:13:59.501428854Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:14:00.151617809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:14:00.157887831Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:14:02.036423958Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T22:14:04.187792754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:05.454952635Z caller=checkpoint.go:372 msg="writing series" size="393 kB"
level=debug ts=2026-05-12T22:14:08.871716634Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T22:14:09.187164195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:14:09.187180291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:14:09.195154672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:14:11.340190481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:14:12.289684515Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=debug ts=2026-05-12T22:14:14.072603405Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:14:14.074702263Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:14:14.187603782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:14:14.187616031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:14.187643009Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:14:14.187653771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:14:15.710006197Z caller=checkpoint.go:372 msg="writing series" size="668 kB"
level=debug ts=2026-05-12T22:14:15.710029998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-12T22:14:19.125302009Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-12T22:14:19.187547862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:22.373326731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:14:22.554489929Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-12T22:14:22.554513321Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-12T22:14:23.633925759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:14:23.645685289Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:14:24.187648077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:25.960275312Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-12T22:14:29.187862133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:29.380253116Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-12T22:14:29.380286101Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=debug ts=2026-05-12T22:14:32.796149457Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:14:34.187784725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:36.214528698Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=warn ts=2026-05-12T22:14:39.187802869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:39.631851022Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=info ts=2026-05-12T22:14:40.937734092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:14:40.945167251Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-12T22:14:41.342984637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:14:42.1549212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:14:42.163187902Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:14:43.049439194Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=info ts=2026-05-12T22:14:44.187338437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:14:44.187372073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:14:44.187369805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:14:44.187383052Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-12T22:14:44.981984767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37468"
level=debug ts=2026-05-12T22:14:46.466508455Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-12T22:14:47.259646465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:14:47.261256545Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:14:47.287554228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:14:47.291921898Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:14:47.928834134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:14:47.938153665Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:14:49.187460946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:49.884284278Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-12T22:14:51.992122899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:14:53.024415512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:14:53.033289946Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:14:53.302364307Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:14:54.187707838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:14:56.155858358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:14:56.163146993Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:14:56.719760095Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T22:14:59.187338238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:00.138285148Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-12T22:15:00.138307769Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-12T22:15:03.558106445Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-12T22:15:03.558130237Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=1
level=warn ts=2026-05-12T22:15:04.187884024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:15:05.286264585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54586"
level=info ts=2026-05-12T22:15:05.824123536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:15:05.832934449Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:15:06.973337861Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T22:15:09.187586349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:10.391060244Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T22:15:11.345862971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:15:13.808470568Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T22:15:14.074136106Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:15:14.074615644Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=debug ts=2026-05-12T22:15:14.075966322Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:15:14.076022514Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976248 n=1
level=debug ts=2026-05-12T22:15:14.076134117Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976248 n=1
level=info ts=2026-05-12T22:15:14.187521354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:15:14.187554392Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:15:14.187566007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:15:14.187554277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:15:14.98303624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33402"
level=debug ts=2026-05-12T22:15:17.22603666Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-12T22:15:19.187553853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:20.644760575Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=debug ts=2026-05-12T22:15:24.061738218Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T22:15:24.187124391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:25.669377185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:15:27.098466257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:15:27.103986181Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:15:27.479218279Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-12T22:15:29.187030105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:29.345810739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:15:29.889893569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:15:29.899290354Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:15:30.204478838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:15:30.212279442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:15:30.212304832Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:15:30.212338593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:15:30.219531342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:15:30.896838503Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-12T22:15:34.187308838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:34.315375601Z caller=checkpoint.go:372 msg="writing series" size="705 kB"
level=debug ts=2026-05-12T22:15:34.315397501Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=debug ts=2026-05-12T22:15:37.733366288Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-05-12T22:15:38.114433224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:15:38.121937821Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:15:39.187608552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:41.150974355Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
ts=2026-05-12T22:15:41.347801115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:15:43.357676044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:15:43.362619293Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:15:44.187738766Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:15:44.187766887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:44.187782056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:15:44.187790237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:15:44.47992346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:15:44.497797982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:15:44.569547689Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-12T22:15:47.986276604Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:15:47.986302337Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-12T22:15:49.187775537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:51.403362649Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=info ts=2026-05-12T22:15:51.893949395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:15:51.904543205Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:15:54.187319546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:54.823406657Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:15:54.82342888Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=debug ts=2026-05-12T22:15:58.23911996Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:15:59.172607056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:15:59.180940052Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:15:59.187997684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:00.011762379Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:16:01.656518273Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:16:04.18789897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:05.075185989Z caller=checkpoint.go:372 msg="writing series" size="501 kB"
level=debug ts=2026-05-12T22:16:05.268747935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:16:05.288913894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52416"
level=debug ts=2026-05-12T22:16:08.492917539Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:16:08.492938794Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-12T22:16:09.187748093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:16:10.065833243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:16:10.071256084Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-12T22:16:11.349629805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:16:11.909771738Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:16:14.072544213Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:16:14.074649127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:16:14.187715157Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:16:14.187747287Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:16:14.187765469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:16:14.187762563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:15.327539178Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:16:15.88018842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:16:15.887651251Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:16:18.745677021Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:16:18.745700181Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-05-12T22:16:19.187182089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:16:21.803034905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:16:21.811766691Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:16:22.218527216Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-05-12T22:16:22.218565297Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=1
level=info ts=2026-05-12T22:16:23.532427294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:16:23.538512333Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:16:24.187669886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:25.581280482Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-12T22:16:28.998633222Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-12T22:16:29.188114407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:31.845444582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:16:32.41700791Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:16:32.41703154Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-05-12T22:16:34.187667567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:35.833971674Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:16:38.62729881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:16:38.632987883Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:16:39.187811595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:39.25220398Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
ts=2026-05-12T22:16:41.351403293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:16:42.346127072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:16:42.354353255Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:16:42.669434175Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-12T22:16:44.187554695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:16:44.187562643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:44.187582159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:16:44.187590172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:16:46.087656Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-12T22:16:46.915880574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:16:46.923955828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:16:49.187629085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:49.506043064Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-12T22:16:49.506089713Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-05-12T22:16:50.241813094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:16:50.246181191Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:16:52.923063276Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:16:54.187370536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:56.340005176Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-12T22:16:58.601392009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:16:58.609060788Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:16:59.187840756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:59.758877983Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:16:59.758899066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-12T22:17:00.949822977Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:17:03.176180425Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-12T22:17:03.836431785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:17:04.187386529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:04.545322272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:17:04.552370862Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:17:04.552393067Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:17:04.552416572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:17:04.557941343Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:17:06.593435333Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=warn ts=2026-05-12T22:17:09.187963737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:10.012522014Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:17:10.012542813Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:17:11.353544275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:17:13.429315618Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T22:17:14.072984072Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:17:14.074015568Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:17:14.187895832Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:17:14.187921107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:17:14.187928955Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:17:14.187947754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:17:14.991414653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60432"
level=debug ts=2026-05-12T22:17:16.846290207Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-12T22:17:18.929717366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:17:18.939377695Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:17:19.187273232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:20.26517084Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:17:21.907530423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:17:21.91247063Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:17:21.912492599Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:17:21.912516285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:17:21.91627742Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:17:22.530080874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:17:22.535355542Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:17:23.681674745Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T22:17:24.187099178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:27.100025892Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=debug ts=2026-05-12T22:17:27.100049231Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=5
level=info ts=2026-05-12T22:17:27.584047628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:17:27.591791634Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:17:29.187792836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:30.370420148Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:17:30.517869431Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:17:33.935834508Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-12T22:17:34.186980133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:35.230010115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:17:35.236836655Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:17:37.352723458Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T22:17:39.187986242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:40.281668197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:17:40.289847554Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:17:40.771220668Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-12T22:17:41.335745387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:17:41.343709529Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:17:41.355988988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:17:44.187548932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:17:44.187580871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:17:44.187589473Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:17:44.187639293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:44.188663577Z caller=checkpoint.go:384 msg="flushing series" totalSize="320 kB" series=4
level=info ts=2026-05-12T22:17:44.191810936Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-12T22:17:44.194529981Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006656276s
ts=2026-05-12T22:17:44.993166304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59208"
level=debug ts=2026-05-12T22:17:46.091022901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:17:49.187396737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:50.126653022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:17:50.130936414Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:17:50.130956394Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:17:50.130979315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:17:50.13457115Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:17:53.326886766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:17:53.332161536Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:17:54.18701161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:17:54.85449153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:35346"
level=warn ts=2026-05-12T22:17:59.187337389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:02.054669671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:18:02.059525172Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:18:04.094580358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:18:04.187870228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:18:09.187189434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:09.392529236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:18:09.398846446Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-12T22:18:11.356867374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:18:14.072756625Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:18:14.073822364Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:18:14.18771755Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:18:14.187741434Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:18:14.187751321Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:18:14.18775855Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:18:14.187792853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:14.187929139Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-12T22:18:14.187937384Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:18:14.18795482Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.906µs success=true
level=debug ts=2026-05-12T22:18:14.188224574Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:18:14.757763232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:18:14.76641437Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:18:17.521844253Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:18:19.18747301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:20.712332843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:18:20.719894217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:18:20.855197769Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-12T22:18:23.772122191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:18:23.780858259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:18:24.187089057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:24.189266297Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-12T22:18:24.971555571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:18:24.979521196Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:18:27.522482659Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T22:18:29.187751439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:30.855581283Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-12T22:18:34.187793169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:34.188921386Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-12T22:18:37.26926157Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:18:37.522124201Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-12T22:18:37.670790733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:18:37.672647593Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:18:39.187677051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:40.857753715Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-12T22:18:40.857777987Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=9
ts=2026-05-12T22:18:41.358104014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:18:42.36698696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:18:42.373276527Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:18:44.187199722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:18:44.187208253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:44.18722925Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:18:44.187237812Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:18:44.188860347Z caller=checkpoint.go:372 msg="writing series" size="812 kB"
ts=2026-05-12T22:18:44.995372059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47776"
level=debug ts=2026-05-12T22:18:47.52162288Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:18:49.188045774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:49.357178676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:18:49.362890377Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:18:50.85519987Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:18:50.855221614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-12T22:18:54.187059434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:54.189273508Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-12T22:18:55.884360804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:18:55.889598369Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:18:57.522226447Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-12T22:18:59.187926628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:00.855963325Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T22:19:04.187086276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:04.189656367Z caller=checkpoint.go:372 msg="writing series" size="683 kB"
level=info ts=2026-05-12T22:19:05.575283458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:05.583360802Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:05.583379555Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:19:05.583412682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:05.591009128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:19:06.018127209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:06.026906796Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:19:07.521651488Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-12T22:19:07.521679946Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-12T22:19:07.57190521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:19:09.187633109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:10.855670456Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-12T22:19:11.360344496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:19:13.245816556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:19:13.25439023Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:19:14.073162541Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:19:14.074240648Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:19:14.18712887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:19:14.187138453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:14.187158433Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:19:14.18716671Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:19:14.188656317Z caller=checkpoint.go:372 msg="writing series" size="789 kB"
level=debug ts=2026-05-12T22:19:17.521665385Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-12T22:19:18.699943101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:18.708376657Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:19:19.187753259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:19:19.894883574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:19:19.901802093Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:19:20.865762253Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-12T22:19:20.865785996Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-12T22:19:24.186985792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:24.189316597Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-12T22:19:27.524487645Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:19:27.524512923Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-05-12T22:19:29.187554243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:30.855245652Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-12T22:19:34.021347281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:19:34.187597204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:34.188901463Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-05-12T22:19:36.043044975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:19:36.047410454Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:19:37.215180323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:19:37.522248726Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:19:38.20011394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:19:38.204116435Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:19:38.907155395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:19:38.912330512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:19:39.187286352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:40.85553677Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
level=debug ts=2026-05-12T22:19:40.855559625Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-12T22:19:41.363375234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:19:44.186942478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:19:44.186993175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:19:44.18701145Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:19:44.187051909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:44.18916219Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-12T22:19:47.522558192Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-12T22:19:48.375998935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:48.386776703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:19:49.18799435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:50.855013852Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-12T22:19:54.187588646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:54.188819381Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:19:54.340437229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:54.349435501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:54.349454956Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:19:54.349478928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:54.357004426Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:19:54.866907274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50356"
level=debug ts=2026-05-12T22:19:57.522061792Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T22:19:59.187439306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:00.855202692Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-12T22:20:04.187897077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:04.191282134Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-12T22:20:04.191300858Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=7
ts=2026-05-12T22:20:05.312238375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37744"
level=info ts=2026-05-12T22:20:05.686767979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:20:05.696529226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:20:06.024393689Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:20:07.522451075Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-12T22:20:08.426065748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:20:08.434140731Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:20:08.43416634Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:20:08.43419226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:20:08.440994216Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:20:08.965413862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:20:08.973506259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:20:09.188003003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:10.85516865Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-12T22:20:11.365139616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:20:14.072608914Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:20:14.074730596Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:20:14.187611834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:20:14.187630522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:14.187642992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:20:14.187652507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:20:14.188754095Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T22:20:15.001541464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57328"
level=debug ts=2026-05-12T22:20:17.521734464Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-12T22:20:19.187327376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:20.855104592Z caller=checkpoint.go:372 msg="writing series" size="454 kB"
level=info ts=2026-05-12T22:20:21.849578091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:20:21.854479846Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:20:22.689140567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:20:23.424983731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:20:23.432165488Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:20:24.187972594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:24.189003712Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=debug ts=2026-05-12T22:20:27.521881506Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=warn ts=2026-05-12T22:20:29.187887764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:20:29.791963417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:20:29.799607732Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:20:30.855544042Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-12T22:20:30.855574805Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-12T22:20:32.206523304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:20:32.211287911Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:20:34.187742386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:34.188770586Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-12T22:20:37.522334434Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-05-12T22:20:37.739860001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:20:37.746585317Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:20:38.831671749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:20:38.831694913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:20:38.836270874Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:20:39.187880977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:40.855278912Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=debug ts=2026-05-12T22:20:40.85535692Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:20:40.90444932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:20:40.912381359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:20:41.366015208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:20:44.187586786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:20:44.187617311Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:20:44.187624921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:20:44.187681702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:44.188792858Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-12T22:20:47.522353042Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-12T22:20:49.188005754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:50.856176645Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:20:53.902737435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:20:53.910722575Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:20:54.187214084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:54.188541619Z caller=checkpoint.go:372 msg="writing series" size="363 kB"
level=debug ts=2026-05-12T22:20:57.5237351Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:20:57.523758191Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=5
level=warn ts=2026-05-12T22:20:59.186990391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:00.855339183Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-12T22:21:01.110120415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:21:01.329683081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:21:01.336819119Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:21:01.481418418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:01.487933001Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:21:04.114860569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:21:04.119426133Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:21:04.187733141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:04.188932822Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-12T22:21:07.522490422Z caller=checkpoint.go:372 msg="writing series" size="585 kB"
level=warn ts=2026-05-12T22:21:09.187547636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:10.855928455Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:21:10.855956629Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-12T22:21:11.002455253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:21:11.367956871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:21:14.072859318Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:21:14.073907192Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:21:14.187854851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:21:14.187885606Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:21:14.18789385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:21:14.187906782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:14.189174046Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:21:17.522351746Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:21:19.11864471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:21:19.124554356Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:21:19.187829737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:20.855379432Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:21:20.855401024Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-12T22:21:21.06984933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:21:21.077945314Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:21:22.909697259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:21:22.915729703Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:21:24.187289818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:24.240208116Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:21:24.24023461Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
level=info ts=2026-05-12T22:21:27.044224769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:21:27.05268786Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:21:27.522336931Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-12T22:21:29.187772646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:30.85582046Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-12T22:21:32.108584351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:21:32.114732367Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:21:34.187242729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:34.189896144Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:21:34.189911849Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=debug ts=2026-05-12T22:21:37.521626579Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T22:21:39.188092106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:40.855603702Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-05-12T22:21:41.371490333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:21:42.330120489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:21:44.187095782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:21:44.187132112Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:21:44.187142686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:21:44.187142042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:44.188300071Z caller=checkpoint.go:372 msg="writing series" size="8.5 kB"
level=debug ts=2026-05-12T22:21:47.522561775Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-12T22:21:48.132346171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:48.139257012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:48.139280268Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:21:48.139308939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:48.146714312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:21:48.365124055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:21:49.187406054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:21:50.239192313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:21:50.24642047Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:21:50.855942974Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:21:50.855965889Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=warn ts=2026-05-12T22:21:54.187016168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:54.189189118Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-12T22:21:54.880597717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59928"
level=info ts=2026-05-12T22:21:54.914461181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:21:54.923829952Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:21:57.522585819Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-12T22:21:59.187594787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:00.85643792Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:22:00.856459389Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:22:04.090244383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:22:04.098427627Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:22:04.18764188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:04.188813032Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T22:22:06.095398885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:22:06.103176282Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:22:07.522125485Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=warn ts=2026-05-12T22:22:09.187755002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:22:10.188906637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:22:10.194775281Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:22:10.856113879Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:22:10.856139832Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-12T22:22:11.374270071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:22:13.387467771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:22:13.394598536Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:22:14.072694261Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:22:14.074813238Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:22:14.187725056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:22:14.187757952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:22:14.18776926Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:22:14.187786521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:14.189088078Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=info ts=2026-05-12T22:22:16.249189626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:22:16.257696616Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:22:17.522269582Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-12T22:22:17.604617183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:22:19.187710511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:20.855676198Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-12T22:22:22.589108073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:22:22.590763351Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:22:24.187842681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:24.188945811Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-12T22:22:24.883387923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53102"
level=debug ts=2026-05-12T22:22:27.522312957Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=debug ts=2026-05-12T22:22:27.522335598Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-12T22:22:29.187701155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:30.855462062Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:22:31.307981679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:22:31.314993317Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:22:34.187689315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:34.188795649Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-12T22:22:37.522301765Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T22:22:39.187499015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:22:40.284588042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:22:40.294452604Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:22:40.443671488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:22:40.448365306Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:22:40.855615941Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-05-12T22:22:41.376638689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:22:42.504062022Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:22:42.504111173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:22:42.508391168Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:22:44.187929044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:22:44.187963561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:22:44.187973945Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:22:44.18797229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:44.189040386Z caller=checkpoint.go:384 msg="flushing series" totalSize="333 kB" series=4
level=info ts=2026-05-12T22:22:44.192235168Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-12T22:22:44.194111128Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005948398s
level=info ts=2026-05-12T22:22:45.42691633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:22:45.43460532Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:22:49.187190725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:22:54.187735153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:22:59.188064502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:22:59.477184001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:22:59.483830647Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:23:02.363454985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:23:02.372878044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:23:03.772643011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:23:03.777367799Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:23:04.187884891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:23:05.329380529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35992"
level=info ts=2026-05-12T22:23:06.096282575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:23:06.1045491Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:23:08.173114549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:23:08.17938829Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:23:09.187548952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:23:11.377847943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:23:12.886810851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:23:14.07212013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:23:14.07423022Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:23:14.187331898Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:23:14.187345989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:23:14.187384364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:23:14.187400306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:23:14.187408485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:23:14.187506965Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-12T22:23:14.187515782Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:23:14.187532782Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.83µs success=true
level=debug ts=2026-05-12T22:23:14.187704602Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:23:17.136974702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:23:17.141436534Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:23:17.521791406Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=debug ts=2026-05-12T22:23:17.609346716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:23:19.187788276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:20.855511182Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-12T22:23:22.496277292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:23:22.502377678Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:23:22.502399275Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:23:22.502421697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:23:22.506290457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:23:22.777023853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:23:22.785339909Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:23:24.187765742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:24.187820014Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-12T22:23:27.522248167Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:23:27.891200766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:23:27.8990571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:23:29.18806306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:23:30.122229731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:23:30.126572014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:23:30.855502432Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-05-12T22:23:34.187268983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:34.188403927Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-12T22:23:37.522022352Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:23:39.187010771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:40.857494175Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:23:40.857516404Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=9
level=info ts=2026-05-12T22:23:41.235307633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:23:41.243477892Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-12T22:23:41.379252974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:23:44.187248087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:23:44.187257474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:44.187281035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:23:44.187289839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:23:44.189244858Z caller=checkpoint.go:372 msg="writing series" size="832 kB"
ts=2026-05-12T22:23:45.018894695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60314"
level=debug ts=2026-05-12T22:23:47.209683313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:23:47.521588359Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T22:23:48.60655826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:23:48.615339599Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:23:49.187740693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:50.854949797Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:23:50.854973874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-12T22:23:54.187962727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:54.188114462Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-12T22:23:56.416011799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:23:56.421007827Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:23:57.521186971Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:23:58.57729397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:23:58.584376359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:23:58.639473053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:23:58.643362476Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:23:59.187640332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:00.854836188Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-12T22:24:02.204145795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:24:04.187421274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:04.188614403Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=info ts=2026-05-12T22:24:04.481600878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:24:04.488860336Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:24:04.488884101Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:24:04.488913604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:24:04.493919241Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:24:06.037999272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:24:06.042298631Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:24:07.521537455Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T22:24:09.18762114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:24:10.803981336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:24:10.812056435Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:24:10.855175366Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-12T22:24:11.281874191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:24:11.291213393Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:24:11.380174914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:24:14.072065221Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:24:14.074147166Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:24:14.186949049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:24:14.186977491Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:24:14.186987033Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:24:14.187019004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:14.188588719Z caller=checkpoint.go:372 msg="writing series" size="995 kB"
level=debug ts=2026-05-12T22:24:14.18860156Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=info ts=2026-05-12T22:24:15.450750375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:24:15.450840171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-12T22:24:15.454376882Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:24:15.751243163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:24:15.754893378Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:24:16.051673994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="48 kB" avg_uncomp="48 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:24:16.055257503Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:24:16.351199813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:24:16.354789193Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:24:16.65157942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:24:16.65511503Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:24:16.951053539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="853 B" avg_uncomp="853 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:24:16.954577693Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:24:17.0731156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:24:17.078912134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:24:17.25120539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:24:17.254642241Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:24:17.521572553Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T22:24:19.18720069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:20.865871944Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-12T22:24:20.865894873Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=warn ts=2026-05-12T22:24:24.1869922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:24.188303386Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=debug ts=2026-05-12T22:24:27.522427491Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:24:27.522450951Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=2
level=warn ts=2026-05-12T22:24:29.187653528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:24:30.024766227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:24:30.031745274Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:24:30.855046388Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:24:31.371802827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:24:31.375358844Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:24:31.881664179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:24:31.884954053Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:24:32.603929193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:24:32.607779931Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:24:32.683103768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:24:32.686356703Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:24:33.243684589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:24:33.247352481Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:24:33.381865657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:24:33.385712493Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:24:34.187936293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:34.188150343Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
ts=2026-05-12T22:24:35.334874039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43726"
level=info ts=2026-05-12T22:24:35.55112184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:24:35.556577808Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:24:37.521238775Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-12T22:24:39.187484736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:24:39.590794307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:24:39.598187567Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:24:40.547116284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:24:40.855054648Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
level=debug ts=2026-05-12T22:24:41.219090863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:24:41.38279628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:24:42.314889482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:24:42.319018043Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:24:44.187284234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:24:44.18730325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:24:44.187326784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:24:44.187336146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:24:44.188465541Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
ts=2026-05-12T22:24:45.023440367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53206"
level=debug ts=2026-05-12T22:24:47.521294953Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-12T22:24:47.521323168Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-12T22:24:47.785004458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:24:47.789477234Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:24:48.896646932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:24:48.901769752Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=warn ts=2026-05-12T22:24:49.187944735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:50.854767318Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-12T22:24:51.353440145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:24:51.360547526Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:24:53.441120757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:24:53.444987148Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:24:54.186996791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:54.18808174Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:24:54.714919873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:24:54.719645532Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:24:54.719666371Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:24:54.719690003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:24:54.723483459Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-12T22:24:54.899564331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56216"
level=info ts=2026-05-12T22:24:55.278351108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:24:55.281893626Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:24:56.816031874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:24:56.819409792Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:24:56.990033455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:24:56.993715495Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:24:57.521363856Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T22:24:59.187530994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:00.855158655Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-12T22:25:01.603406403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:25:01.611377033Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:25:02.791608622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:25:02.795315128Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:25:04.187122431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:04.190853721Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-12T22:25:04.190869562Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
level=info ts=2026-05-12T22:25:04.264548106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:25:04.272789288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:25:05.738647022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:25:05.749351594Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:25:06.686182008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:25:06.689680588Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:25:07.521286313Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T22:25:09.187895249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:10.854716771Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-12T22:25:11.263267702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:25:11.267498305Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-12T22:25:11.385993885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:25:14.072650659Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:25:14.074771109Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:25:14.123322333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:25:14.187705612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:25:14.187739035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:25:14.187749808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:25:14.187755176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:14.187788673Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T22:25:15.025172808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33722"
level=debug ts=2026-05-12T22:25:17.521784529Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-12T22:25:19.187984481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:20.85541366Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=info ts=2026-05-12T22:25:22.939120758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:25:22.944910785Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:25:23.064555802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:25:24.187289292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:24.18844848Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=info ts=2026-05-12T22:25:24.359768329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:25:24.363983589Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:25:27.521235528Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=warn ts=2026-05-12T22:25:29.187296556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:30.855451539Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-12T22:25:31.907711921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:25:31.91858284Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:25:34.187785936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:34.187970133Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-12T22:25:37.521418129Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=debug ts=2026-05-12T22:25:37.521439501Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=warn ts=2026-05-12T22:25:39.187833075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:40.201614827Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:25:40.85565284Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-05-12T22:25:41.02777703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:25:41.033361684Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-12T22:25:41.388722635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:25:42.621796352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:25:42.631800194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:25:42.631823635Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:25:42.631856224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:25:42.636299828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:25:42.638988033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:25:42.638999359Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:25:44.187667098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:25:44.187697299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:25:44.187706602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:25:44.187701621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:44.187796743Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-12T22:25:44.501393593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:25:44.506558693Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:25:45.312709328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:25:45.316540343Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:25:47.357630843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:25:47.361114426Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:25:47.521483942Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-12T22:25:47.738201696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:25:47.742489781Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:25:48.127677385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:25:48.130907148Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:25:49.041425587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:25:49.044504165Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:25:49.187980567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:25:50.0180113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:25:50.021690371Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:25:50.855353576Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:25:50.855377424Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-12T22:25:54.187221475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:54.188407219Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-12T22:25:55.26302024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:25:55.271728961Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:25:57.522512591Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:25:57.52253546Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=2
level=info ts=2026-05-12T22:25:57.743124298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:25:57.750575461Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:25:58.293493494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:25:58.301008261Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:25:58.817358436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:25:58.820715311Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:25:59.187855878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:00.854762116Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-12T22:26:04.18779255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:04.187927571Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-12T22:26:07.521942233Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-12T22:26:08.78605659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:26:08.788042996Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:26:09.187444779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:10.855015468Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
ts=2026-05-12T22:26:11.390884684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:26:12.221762292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="154 kB" avg_uncomp="154 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:26:12.221802518Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-12T22:26:12.227938945Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:26:14.072599075Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:26:14.07472025Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:26:14.187739305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:26:14.187739976Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:26:14.187763535Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:26:14.1877713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:26:14.18793584Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:26:14.187949165Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-12T22:26:17.231669952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:26:17.23595831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:26:17.521880349Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-12T22:26:19.187975537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:20.854970336Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=info ts=2026-05-12T22:26:23.230369399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:26:23.234748302Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:26:24.187285853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:24.240910358Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:26:24.24093775Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=info ts=2026-05-12T22:26:24.741117449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:26:24.746945167Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:26:26.513435124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:26:26.518356626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:26:26.651609527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:26:26.659830303Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:26:26.659851467Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:26:26.659876672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:26:26.667528415Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:26:26.85498104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:26:26.858777322Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:26:27.522184413Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-12T22:26:28.425371585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:26:28.429512842Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:26:29.18749179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:26:29.898174232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:26:29.901433202Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:26:30.117683422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:26:30.125798481Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:26:30.854824939Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-12T22:26:31.74036957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:26:31.74378961Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:26:34.187312606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:34.189777553Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:26:34.189796339Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-12T22:26:36.056511075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:26:36.061520928Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:26:37.521698539Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:26:37.792416089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:26:37.800639967Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:26:37.800665525Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:26:37.800690338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:26:37.808323006Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:26:38.310787198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:26:38.314221921Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:26:39.187982844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:26:39.856363828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:26:39.864205422Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:26:40.854586573Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-12T22:26:41.100949895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:26:41.108243978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:26:41.392337357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:26:44.187850234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:26:44.1878807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:26:44.187888966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:26:44.18788968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:44.187956401Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-12T22:26:45.031754277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33742"
level=info ts=2026-05-12T22:26:46.006285589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:26:46.010752931Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:26:46.850289165Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:26:47.521455245Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-12T22:26:47.812285394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:26:47.816385077Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:26:47.845452273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:26:47.848623432Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:26:49.187005785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:50.856569062Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-12T22:26:50.856589099Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=info ts=2026-05-12T22:26:52.061655685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:26:52.069566587Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:26:54.187766633Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:26:54.187781182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:26:55.891347919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:26:55.894810388Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:26:57.132634588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:26:57.521597415Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-12T22:26:59.188017759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:00.855790365Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:27:00.855815768Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:27:02.887185929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:27:02.891405243Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:27:04.187994458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:04.188023578Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T22:27:07.051846501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:27:07.057694662Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:27:07.057716814Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:27:07.05774011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:27:07.061426784Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:27:07.522373417Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=warn ts=2026-05-12T22:27:09.187758343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:10.855909419Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:27:10.855933096Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-12T22:27:11.394480288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:27:12.620400865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:27:12.630119236Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:27:14.073205036Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:27:14.074269018Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:27:14.187317442Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:27:14.187344079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:27:14.187353622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:27:14.187350206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:14.188783957Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
ts=2026-05-12T22:27:15.032729894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60062"
level=debug ts=2026-05-12T22:27:15.462011474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:27:17.521749067Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-12T22:27:17.564994477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:27:17.574503606Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:27:18.833899383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:27:18.837289867Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:27:19.187657218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:19.609703472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:27:19.61526373Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:27:20.673113984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:27:20.682120669Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:27:20.855521238Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-12T22:27:21.144475908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:27:21.148493301Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:27:21.154540369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:27:21.157783569Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:27:22.949727549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:27:22.953701679Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:27:24.053836276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:27:24.057213095Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:27:24.187654838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:24.187724065Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=debug ts=2026-05-12T22:27:27.522205619Z caller=checkpoint.go:372 msg="writing series" size="370 kB"
level=info ts=2026-05-12T22:27:27.686489599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:27:27.691108218Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:27:29.188005099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:30.854486206Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:27:33.973815769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:27:33.977703141Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:27:34.187948925Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T22:27:34.187951526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:35.22512364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:27:35.23223982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:27:36.228854893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:27:36.232739711Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:27:37.27075721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:27:37.277289485Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:27:37.521960386Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-12T22:27:37.521981486Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-12T22:27:39.187313263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:40.854547811Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-12T22:27:41.045257058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:27:41.049212081Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:27:41.397501138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:27:41.744731229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:27:44.186959461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:27:44.187003717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:27:44.187013992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:27:44.187044455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:44.18809924Z caller=checkpoint.go:384 msg="flushing series" totalSize="200 kB" series=1
level=info ts=2026-05-12T22:27:44.193260094Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-12T22:27:44.194799042Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007139691s
ts=2026-05-12T22:27:45.033572556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34590"
level=debug ts=2026-05-12T22:27:47.664088782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:27:49.187127347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:50.369788124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:27:50.374524038Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:27:52.118505725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:27:52.12312229Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:27:53.20065802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:27:53.20457323Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:27:54.187304213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:27:59.187724909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:00.234566342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:28:00.24317181Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:28:03.175818075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:28:03.179329287Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:28:04.187910207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:06.211280742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:28:06.215152167Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:28:06.402682497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:28:06.408247441Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=info ts=2026-05-12T22:28:07.347637164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:28:07.353408094Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:28:08.314366458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:28:08.322043666Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:28:09.188035801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:11.275942876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:28:11.285190834Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:28:11.400629489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:28:12.92444711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:28:12.924464036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:28:12.931844968Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:28:14.072665864Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:28:14.07477136Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:28:14.187659873Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:28:14.187669942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:14.187695607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:28:14.187719506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:28:14.187730392Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:28:14.187797758Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-12T22:28:14.187805908Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:28:14.187823675Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.604µs success=true
level=debug ts=2026-05-12T22:28:14.188024877Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-05-12T22:28:15.034808016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43084"
level=info ts=2026-05-12T22:28:17.186795026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:28:17.190652405Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:28:17.522376232Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:28:18.981327268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:28:18.984684088Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:28:19.187944564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:20.537108307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:28:20.541362902Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:28:20.855304408Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-12T22:28:22.014206316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:28:23.450030194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:28:23.45350057Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:28:23.938791469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:28:23.942180243Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:28:24.187411314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:24.188459085Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-12T22:28:25.375283613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:28:25.381315387Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:28:26.219831833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:28:26.228354854Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:28:27.522338618Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:28:28.329498896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:28:28.336818685Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:28:28.336841223Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:28:28.336865524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:28:28.343340862Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:28:29.187564005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:30.854972079Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-12T22:28:33.277792597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:28:33.28226941Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:28:34.187198612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:34.188353433Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-12T22:28:37.522549763Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:28:39.187484154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:40.856506842Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-12T22:28:40.856528593Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=9
ts=2026-05-12T22:28:41.403144577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:28:43.862122223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:28:43.872872761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:28:44.18745405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:28:44.187490756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:28:44.187499824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:28:44.187504802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:44.189339498Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-05-12T22:28:46.728530418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:28:47.323118688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:28:47.332566978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:28:47.444799539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:28:47.448872315Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:28:47.522192577Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:28:49.187184187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:50.262544323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:28:50.266925839Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:28:50.855447001Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:28:50.855471059Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:28:51.40621233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:28:51.409942007Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:28:53.006788203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:28:53.014205102Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:28:53.127626853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:28:53.131331582Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:28:54.1873246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:54.188432977Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
ts=2026-05-12T22:28:54.925343457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54048"
level=debug ts=2026-05-12T22:28:57.521712523Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-12T22:28:59.187882415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:59.327498594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:28:59.335745704Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:28:59.819184308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:28:59.824905308Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:29:00.854986886Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:29:00.891395708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:29:00.895102669Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:29:04.187853301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:04.18920437Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-12T22:29:05.853425995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:29:05.860635072Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:29:06.042048879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:29:06.045587143Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:29:06.500189443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:29:06.504380869Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:29:07.522209655Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T22:29:08.425036518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:29:08.428558418Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:29:09.187451679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:29:09.836515721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:29:09.840803697Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:29:10.855613997Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-12T22:29:11.404800547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:29:11.614145454Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:29:11.801677775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:29:11.805601268Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:29:12.824356499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:29:12.828523572Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:29:14.072291696Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:29:14.074430487Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:29:14.187308186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:29:14.187337394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:29:14.18734606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:29:14.187342261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:14.188921435Z caller=checkpoint.go:372 msg="writing series" size="867 kB"
level=debug ts=2026-05-12T22:29:14.188940094Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
ts=2026-05-12T22:29:15.03775996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56252"
level=debug ts=2026-05-12T22:29:15.953331797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:29:17.522120495Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-12T22:29:19.187199677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:29:19.627360149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:19.631789918Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:19.631807972Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:29:19.631829088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:19.635254179Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:29:20.868187168Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-12T22:29:20.868208346Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-12T22:29:21.678423498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.8 MB" avg_comp="2.5 MB" total_uncomp="87 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:29:21.68567627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:29:24.187902313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:24.188055278Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-05-12T22:29:24.928800872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:37558"
level=debug ts=2026-05-12T22:29:27.523314222Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:29:27.523336642Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=warn ts=2026-05-12T22:29:29.187293202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:30.854966035Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:29:30.925390086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:29:30.929434975Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:29:34.121245133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:34.125367226Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:29:34.187614209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:34.189167287Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=debug ts=2026-05-12T22:29:37.522337022Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:29:38.128101006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:29:38.131827047Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:29:39.187919528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:29:39.913463858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:29:39.918876719Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=debug ts=2026-05-12T22:29:40.854985881Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:29:41.406655117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:29:43.562263018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:29:43.56569734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:29:44.186965946Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:29:44.18699314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:44.187010133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:29:44.187022988Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:29:44.188118483Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-12T22:29:44.300812284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:29:44.308532554Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:29:44.390586843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:29:44.394530245Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-12T22:29:45.038829694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51920"
level=debug ts=2026-05-12T22:29:45.666183371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:29:46.526807423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:29:46.534934805Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:29:46.727514907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:29:46.735349819Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:29:47.253880146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:29:47.257576842Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:29:47.521445595Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-12T22:29:47.521467564Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-12T22:29:49.18756274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:50.855228108Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-12T22:29:51.629480574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:29:51.639640369Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:29:53.507194419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:29:53.510495657Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:29:53.698126997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:29:53.702050212Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:29:54.187797955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:54.188851415Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-12T22:29:54.932171023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60520"
level=info ts=2026-05-12T22:29:56.023330955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:29:56.027809713Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:29:56.027828571Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:29:56.027851933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:29:56.031662331Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:29:57.52231743Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T22:29:59.187456205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:29:59.376621569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:29:59.381773133Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:30:00.85584476Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-12T22:30:02.886679551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:30:02.894558464Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:30:03.324811152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:30:03.328405791Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:30:04.187901085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:04.190136373Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-12T22:30:04.190157417Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
level=info ts=2026-05-12T22:30:05.070243158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:30:05.074886309Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:30:06.629421119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:30:06.631200051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:30:07.52174399Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-12T22:30:08.094496713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:30:08.0988005Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:30:09.187186296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:10.854914002Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:30:11.409950287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:30:11.652096843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:30:11.659292766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:30:13.400602778Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:30:14.072368606Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:30:14.074479457Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:30:14.075763265Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:30:14.075799501Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976249 n=1
level=debug ts=2026-05-12T22:30:14.075841062Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976249 n=1
level=info ts=2026-05-12T22:30:14.187227458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:30:14.187252189Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:30:14.187263032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:30:14.187264765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:14.188439564Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-12T22:30:17.52193021Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-12T22:30:19.187604324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:20.855781483Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=warn ts=2026-05-12T22:30:24.187898328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:24.188997755Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=info ts=2026-05-12T22:30:24.686699308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:30:24.698634917Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:30:24.935356032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:42490"
level=info ts=2026-05-12T22:30:25.241614461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:30:25.245761809Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:30:25.245779912Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:25.245799492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:30:25.248721502Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:30:27.522407695Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-12T22:30:29.187808494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:30:29.795854887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:30:29.799720907Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:30:30.855828145Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-12T22:30:30.855850228Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-12T22:30:32.9395318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:32.943242487Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:32.943263501Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:32.943286745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:32.946371053Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:30:34.187342374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:34.188424392Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-12T22:30:34.796657008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:30:34.801107273Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:30:35.261389135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=964e9224f7332480 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:30:35.26640372Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/964e9224f7332480/19e1e1ad697%3A19e1e2fe76e%3A1be3b0b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yhzs\"}"
level=error ts=2026-05-12T22:30:35.266423471Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:35.266445316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:30:35.26936213Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:30:37.52206479Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=info ts=2026-05-12T22:30:38.505586965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:30:38.510552447Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:30:39.187430357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:30:39.455678825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:30:39.459352509Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:30:39.513208151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:30:39.517320649Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:30:39.813632106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:30:39.817504568Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:30:39.817522006Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:39.817542942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:30:39.820605325Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:30:40.826036261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:30:40.83308989Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:30:40.855379244Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=debug ts=2026-05-12T22:30:40.855399969Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:30:41.412189738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:30:43.550204942Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:30:43.580372051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:43.583845714Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:30:44.133770305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:30:44.137016198Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:30:44.187299678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:30:44.187320469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:30:44.187329227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:30:44.187377978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:44.188534921Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-12T22:30:45.409760784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:30:45.41701717Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:30:46.177514972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:30:46.180913122Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:30:46.180934618Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:46.180956277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:30:46.183778498Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:30:46.255890289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:30:46.259173225Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:30:47.521717225Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-12T22:30:48.807418917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:30:48.811327017Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:30:48.811348484Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:48.811370001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:30:48.814543169Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:30:49.187793674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:30:49.540273565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:30:49.546899172Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:30:50.85528581Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:30:54.085030103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:30:54.091737567Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:30:54.18801186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:54.188243105Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=debug ts=2026-05-12T22:30:54.188268961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:30:55.086179822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:30:55.092888203Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:30:56.309516753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:30:56.313374386Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:30:57.400348394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:30:57.403650739Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:30:57.403673267Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:30:57.403698344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:30:57.406869054Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:30:57.524847534Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:30:57.524868942Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-12T22:30:58.725869705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:30:58.729466945Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:30:59.187501839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:31:00.450056191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:31:00.45360997Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:31:00.854898911Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-12T22:31:01.384795069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:31:04.188018598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:04.188192459Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:31:05.347194851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:31:05.357794023Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:31:05.365353832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55432"
level=debug ts=2026-05-12T22:31:07.522230598Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=warn ts=2026-05-12T22:31:09.187907851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:10.854988425Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:31:10.855010978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-12T22:31:11.41397864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:31:12.290401586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:31:12.294687859Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:31:14.072750643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:31:14.073816957Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:31:14.187792216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:31:14.187795071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:14.187822236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:31:14.187831487Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:31:14.18918774Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
ts=2026-05-12T22:31:15.04664765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53374"
level=info ts=2026-05-12T22:31:15.458523031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:31:15.462628493Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:31:16.053054239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:31:16.057201569Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:31:16.145680863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:31:16.150035506Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:31:17.521941469Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:31:18.652384862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:31:18.656172094Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:31:19.085528975Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:31:19.187523349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:20.855560712Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:31:20.855581929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-12T22:31:21.57233455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:31:21.576328295Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:31:23.095259774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:31:23.099192371Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:31:23.863179235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:31:23.866451778Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:31:23.866471866Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:31:23.866496125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:31:23.869347032Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:31:24.18790179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:24.239048316Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:31:24.239089573Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
level=info ts=2026-05-12T22:31:25.713119168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:31:25.723540027Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:31:27.03005237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:31:27.034585236Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:31:27.034607706Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:31:27.034630211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:31:27.038561158Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:31:27.522313692Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-12T22:31:29.187622728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:30.854974566Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-12T22:31:34.11110092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:31:34.114681251Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:31:34.1873067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:34.189631106Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:31:34.189644114Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-12T22:31:34.453301974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:31:34.456741025Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:31:35.392796579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:31:35.396409752Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:31:36.780340824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:31:36.783846987Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:31:37.198024807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:31:37.205180264Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:31:37.522123786Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T22:31:39.187921329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:40.855432334Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-12T22:31:40.868738552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:31:40.875454872Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-12T22:31:41.416330156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:31:43.310099448Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:31:43.434245347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:31:43.437573797Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:31:43.990415511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:31:43.994601681Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:31:44.131793309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:31:44.135700345Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:31:44.187004991Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:31:44.187035289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:31:44.187042856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:31:44.187046045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:44.188149195Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:31:45.471209033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:31:45.479349335Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:31:45.479368819Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:31:45.479405495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:31:45.486951198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:31:47.523169332Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-12T22:31:47.714875097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:31:47.722372701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:31:49.187165853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:50.857827651Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-12T22:31:50.857853112Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=warn ts=2026-05-12T22:31:54.18774901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:54.188827788Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T22:31:54.50842446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:31:57.521998523Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-12T22:31:58.168979671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:31:58.176617898Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:31:59.187253029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:00.855848657Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:32:00.85587004Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:32:02.470379353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:32:02.478511202Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:32:04.187055189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:04.188147796Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-12T22:32:07.522088486Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-12T22:32:09.187702288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:09.28936395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:32:09.297371474Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:32:10.348591768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:32:10.352368226Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:32:10.856388797Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:32:10.856408953Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:32:11.030270288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:32:11.034273609Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-12T22:32:11.418619571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:32:11.914697409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:32:11.917932627Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:32:13.008982347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:32:13.012767482Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:32:13.253503197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:32:13.256823692Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:32:13.286093537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:32:13.28978477Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:32:14.072117343Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:32:14.074299308Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:32:14.186999186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:14.187006517Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:32:14.187029636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:32:14.187038163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:32:14.188544542Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T22:32:17.522405482Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-12T22:32:17.814241224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:32:19.187883161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:20.855592708Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-12T22:32:20.967046768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:32:20.974824362Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:32:22.090115826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:32:22.094044561Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:32:22.848956003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:32:22.8537298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:32:24.187183478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:24.18833086Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
ts=2026-05-12T22:32:24.947602134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41430"
level=info ts=2026-05-12T22:32:26.178971458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:32:26.183062457Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:32:26.183096853Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:32:26.183120579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:32:26.186682538Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:32:26.851418102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:32:26.85879551Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:32:27.522432052Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-12T22:32:27.522456254Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-12T22:32:28.7551867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:32:28.76045873Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:32:29.187889793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:30.855574594Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:32:30.963118692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:32:30.967348318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:32:34.187056313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:34.188187553Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-12T22:32:34.705819878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:32:34.713589406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:32:35.113791929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:32:35.117461525Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:32:37.52219329Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-12T22:32:37.732783671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:32:37.74030355Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:32:39.187332106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:40.688650347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:32:40.692996192Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:32:40.855365673Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
ts=2026-05-12T22:32:41.420320382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:32:44.187888367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:32:44.187914372Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:32:44.187922834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:32:44.187938276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:44.188951013Z caller=checkpoint.go:384 msg="flushing series" totalSize="360 kB" series=4
level=info ts=2026-05-12T22:32:44.1917227Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-12T22:32:44.192869101Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004894624s
ts=2026-05-12T22:32:45.050278455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58164"
level=debug ts=2026-05-12T22:32:46.575706587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:32:49.188105574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:49.924993935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:32:54.177721782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:32:54.183234663Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:32:54.187323028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:32:59.187238183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:59.704950454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:32:59.712329051Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:33:04.18792204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:06.687936187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:33:06.691980646Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:33:08.417456702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:33:08.425102869Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:33:08.42512482Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:33:08.425152253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:33:08.431970584Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:33:09.187196742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:09.296116583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:33:09.304584803Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:33:09.30460243Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:33:09.304624279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:33:09.312392564Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:33:09.947881253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:33:09.952208649Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:33:10.159199355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:33:10.163281673Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:33:10.815044777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:33:10.818468694Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-12T22:33:11.421467368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:33:12.26667895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:33:12.270134945Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:33:12.594905323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:33:12.598237683Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:33:14.072244438Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:33:14.074370318Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:33:14.186850835Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:33:14.186982153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:33:14.187005521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:33:14.187013583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:33:14.187037361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:14.187243296Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-12T22:33:14.187253624Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:33:14.187275781Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.283µs success=true
level=debug ts=2026-05-12T22:33:14.187480241Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:33:14.359231401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:33:14.366421116Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:33:16.893138693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:33:16.896780317Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:33:17.521625269Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:33:18.1233111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:33:18.127115536Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:33:19.187130326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:19.58837621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:33:19.592575594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:33:20.558622769Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:33:20.854550568Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-12T22:33:21.462385645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:33:21.470497759Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:33:23.195605871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:33:23.203767766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:33:24.120478738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:33:24.128409239Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:33:24.187720736Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-12T22:33:24.187746541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:24.235846835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:33:24.239318392Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T22:33:24.954006332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54682"
level=info ts=2026-05-12T22:33:25.71319782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:33:25.720713504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:33:27.522041679Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T22:33:29.187372135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:30.85439502Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-12T22:33:34.187038084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:34.18817115Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:33:34.793749061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:33:34.798105044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:33:34.883110032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:33:34.886340403Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:33:35.101854711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:33:35.379140981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50360"
level=debug ts=2026-05-12T22:33:37.52122593Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:33:39.187162538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:40.857327406Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:33:40.857360142Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=9
ts=2026-05-12T22:33:41.423900755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:33:44.186958981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:33:44.186987962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:33:44.186997139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:33:44.187021539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:44.18841561Z caller=checkpoint.go:372 msg="writing series" size="844 kB"
level=info ts=2026-05-12T22:33:44.850616494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:33:44.85858586Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T22:33:45.053524129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45756"
level=debug ts=2026-05-12T22:33:47.52163959Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:33:49.187778406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:50.855268514Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:33:50.855290962Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:33:51.13223035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:33:51.137115509Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:33:51.137138176Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:33:51.137163515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:33:51.140872144Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:33:51.950542848Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:33:51.950597345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:33:51.955130542Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:33:53.537502164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:33:53.541222846Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:33:53.558260537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:33:53.561416883Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:33:54.187186598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:54.188369067Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-12T22:33:54.619974077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:33:54.624299681Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-12T22:33:54.957096347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57566"
level=info ts=2026-05-12T22:33:57.063117127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:33:57.070264674Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:33:57.521558085Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:33:58.113336093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:33:58.117233723Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:33:59.187600203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:00.855141935Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:34:01.051801407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:34:01.057800089Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:34:03.399590918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:34:03.403278921Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:34:04.186987575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:04.188289116Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=info ts=2026-05-12T22:34:04.295713894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:34:04.299577489Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:34:04.828133594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:34:04.831546623Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:34:06.547449931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:34:06.554648479Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:34:07.521667021Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T22:34:08.756194667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:34:08.760370915Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:34:09.18732061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:10.855243001Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-12T22:34:11.426212221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:34:13.220484429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:34:13.223946542Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:34:13.55201446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:13.558579958Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:34:14.072520805Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:34:14.074629921Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:34:14.187818485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:34:14.18784199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:14.187851037Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:34:14.187868742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:34:14.188375635Z caller=checkpoint.go:372 msg="writing series" size="999 kB"
level=debug ts=2026-05-12T22:34:14.188390947Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=info ts=2026-05-12T22:34:15.492176233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:34:15.496786002Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:34:17.521532578Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-12T22:34:17.529709963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:34:17.529708274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="368 B" avg_comp="368 B" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" idle=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:34:17.533501452Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:34:17.830284078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:34:17.835273464Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:34:18.815800313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:34:18.819183172Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:34:18.941690767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:34:18.949051989Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:34:19.187597452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:20.862383054Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-12T22:34:20.862405469Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=debug ts=2026-05-12T22:34:22.210317219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:34:22.211920105Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:34:24.186998543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:24.188174247Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-12T22:34:26.87117233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:34:26.875765112Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:34:27.522669414Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:34:27.52268961Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=warn ts=2026-05-12T22:34:29.187575892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:30.854951561Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:34:30.928610092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:34:30.931916042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:34:31.219836749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:34:31.225558833Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:34:34.090414461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:34:34.094240434Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:34:34.187769042Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=warn ts=2026-05-12T22:34:34.187909814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:34:35.38213455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41896"
level=info ts=2026-05-12T22:34:35.753653812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:34:35.758409305Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:34:36.436301653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:34:36.440003032Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:34:37.521351877Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:34:37.77602692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:34:37.784144538Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:34:39.186989377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:39.740805368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:34:39.748255772Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:34:40.143651856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:34:40.147225859Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:34:40.855222059Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:34:41.42708572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=warn ts=2026-05-12T22:34:44.187209176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:44.187239071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:34:44.187261303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:34:44.187270256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:34:44.188375543Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=info ts=2026-05-12T22:34:46.375093815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:34:46.37946146Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:34:47.521785572Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-12T22:34:47.52180654Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-12T22:34:47.940905859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:34:47.947898385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:34:49.187916287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:49.373270876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:34:49.389247858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:34:49.392625397Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:34:50.855112142Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-12T22:34:52.607094456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:34:52.611418928Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:34:53.119937756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:34:53.123539686Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:34:53.288117316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:34:53.291549972Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:34:54.187920826Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-12T22:34:54.187952714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:55.368000933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:34:55.372363359Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:34:57.521456671Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-12T22:34:58.70686593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:58.713711468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:58.713732484Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:34:58.713756038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:58.719758601Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:34:59.18735065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:00.855022542Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-12T22:35:01.103965191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:35:01.107591083Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:35:03.553192647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:35:03.56259744Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:35:03.562618382Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:03.562644591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:35:03.569055442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:35:04.187534772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:04.189247513Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-12T22:35:04.189263819Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=5
level=info ts=2026-05-12T22:35:07.299682433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:35:07.303509725Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:35:07.333564669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:35:07.336781539Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:35:07.521737659Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T22:35:09.187945862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:10.854979817Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:35:11.427906644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:35:11.814704823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:35:11.81893369Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:35:14.072453103Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:35:14.074575701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:35:14.187480285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:35:14.187512911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:35:14.187522931Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:35:14.187510867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:14.187547649Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-12T22:35:15.061097569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33326"
level=info ts=2026-05-12T22:35:15.179975787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:35:15.185084119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:35:15.420626534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:35:15.427870735Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:35:16.458196371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:35:16.465808723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:35:17.524381283Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-12T22:35:19.187832778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:20.013976946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:35:20.854660371Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-12T22:35:21.0515033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:35:21.059153069Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:35:21.337123322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:35:21.344739536Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:35:22.529035059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:35:22.532723868Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:35:22.591854743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:35:22.597487755Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:35:22.761357673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:35:22.880736928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:35:22.885130254Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:35:24.187640649Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=warn ts=2026-05-12T22:35:24.187692042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:27.521683439Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-12T22:35:29.188092894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:30.854426048Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-12T22:35:30.983855712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:35:30.988007958Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:35:34.187324058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:34.188416777Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-12T22:35:35.386764703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55596"
level=debug ts=2026-05-12T22:35:37.5219915Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=debug ts=2026-05-12T22:35:37.522013096Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-12T22:35:38.304956181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:35:38.313830508Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:35:39.187839085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:35:39.812118305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:35:39.816215244Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:35:40.582712575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:35:40.59079342Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:35:40.854601223Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-05-12T22:35:41.217546466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:35:41.221199302Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-12T22:35:41.429169633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:35:43.893252744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:35:43.897515972Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:35:44.187150386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:35:44.187169396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:44.187184952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:35:44.187200198Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:35:44.188409517Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=info ts=2026-05-12T22:35:44.792117692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:35:44.798791152Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:35:45.772862401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:35:45.77656786Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:35:46.311134481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:35:46.314873639Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:35:46.754291095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:35:47.3110405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:47.314581857Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:35:47.521047676Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-12T22:35:48.125929686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:35:48.129171335Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:35:49.187872225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:35:49.644306149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:35:49.648033869Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:35:50.854754378Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:35:50.854776841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-12T22:35:54.187457987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:54.187648822Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
level=info ts=2026-05-12T22:35:56.323398394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:35:56.326852755Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:35:57.352105671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:35:57.355818491Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:35:57.52285555Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:35:57.522878234Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=warn ts=2026-05-12T22:35:59.187645859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:00.854977519Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:36:01.346030749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:36:01.349525732Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:36:03.870140162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:36:04.187954268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:04.188103365Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:36:04.258114237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:04.262045409Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:04.262062325Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:36:04.262102149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:04.265470651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:36:04.746521781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:36:04.749899784Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:36:07.521592316Z caller=checkpoint.go:372 msg="writing series" size="480 kB"
level=warn ts=2026-05-12T22:36:09.187349906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:10.85492043Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:36:10.854943672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-12T22:36:11.43041141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:36:11.697118683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:36:11.701283636Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:36:12.370348238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:36:12.37743193Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:36:12.937097674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:36:14.072997539Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:36:14.074046611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:36:14.187841026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:36:14.1878669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:36:14.187876642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:36:14.187871193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:14.188095079Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
ts=2026-05-12T22:36:15.063494351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47506"
level=debug ts=2026-05-12T22:36:17.521811205Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:36:17.568092296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:36:17.575636987Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:36:19.187952497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:36:19.901839132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:36:19.905709121Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:36:20.083171367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:36:20.08758844Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:36:20.08760927Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:36:20.087632037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:36:20.09155588Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:36:20.718123124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:36:20.725226309Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:36:20.855002237Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:36:20.855024946Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-12T22:36:22.909705634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:36:22.917117791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:36:23.125674179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:23.129781938Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:36:24.187378905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:24.235629956Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:36:24.235659296Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
level=debug ts=2026-05-12T22:36:27.520994869Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-12T22:36:28.126664584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:36:28.130664462Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:36:29.18803836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:30.854838785Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=info ts=2026-05-12T22:36:32.664600676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:36:32.668650512Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:36:32.722653641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:36:32.726390141Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:36:33.975948574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:36:33.980744668Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:36:34.187125054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:34.189978683Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:36:34.189994619Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-12T22:36:35.471655559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:36:35.475687805Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:36:37.058524571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:36:37.061855746Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:36:37.521133306Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:36:37.669688147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:36:37.673011731Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:36:38.717213038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:36:38.723020872Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:36:38.817474635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:36:38.821648209Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:36:39.018481841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:36:39.187390385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:40.855088623Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-12T22:36:41.432044344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:36:41.447723441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:36:41.4513508Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:36:42.194648548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:36:42.20285933Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:36:44.187668873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:36:44.187702072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:36:44.187713364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:36:44.187709015Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-12T22:36:44.187708839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:47.52086107Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-12T22:36:47.987298517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:36:49.187247498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:50.856338481Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:36:50.856361661Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=info ts=2026-05-12T22:36:51.031779725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:36:51.035761522Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:36:53.272402154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:36:53.276328113Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:36:54.064919978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:36:54.068967486Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:36:54.187365638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:54.188480003Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:36:55.058746602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:55.063304818Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:36:57.184427544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:36:57.188678518Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:36:57.52174285Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-12T22:36:59.187100016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:00.85648314Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:37:00.856505009Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:37:03.750953719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:37:03.758245747Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:37:04.187774011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:04.187812229Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T22:37:04.826720801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:37:04.830813667Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-12T22:37:05.393089045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45506"
level=info ts=2026-05-12T22:37:05.45787463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:37:05.462533697Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:37:06.771273661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:37:06.77874444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:37:07.521056034Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-05-12T22:37:08.970024942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:37:08.973602523Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:37:09.187864158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:09.677120979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:37:09.685379067Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:37:10.855108342Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:37:10.855133043Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:37:11.433647307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:37:14.072563231Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:37:14.074670712Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:37:14.187663159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:37:14.187689731Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:37:14.187698392Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:37:14.187803768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:14.188097509Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=info ts=2026-05-12T22:37:14.276344563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:37:14.279766916Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:37:14.279784983Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:14.279805647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:37:14.282602293Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:37:14.702339866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:37:14.705628957Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:37:14.705646995Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:14.705669012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:37:14.708696912Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:37:16.549622224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:37:17.521439919Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-12T22:37:17.535543905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:37:17.542815657Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:37:19.188108414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:20.158595303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:37:20.162569007Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:37:20.162593052Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:20.162618516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:37:20.165656612Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:37:20.854466328Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-12T22:37:24.184015756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:37:24.187026113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:37:24.187500168Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:37:24.187521726Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=debug ts=2026-05-12T22:37:24.187540765Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-12T22:37:24.187546236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:37:24.190282135Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:37:24.602434308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:37:24.609881773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:37:24.634878255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:37:24.638093234Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-12T22:37:24.976146652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:47084"
level=debug ts=2026-05-12T22:37:27.521016364Z caller=checkpoint.go:372 msg="writing series" size="411 kB"
level=warn ts=2026-05-12T22:37:29.187198982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:30.855081225Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:37:30.855112165Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=6
level=info ts=2026-05-12T22:37:32.409364833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:37:32.412671273Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:37:32.546048957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:37:32.549374134Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:37:33.441563775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:37:33.445085228Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:37:33.445105145Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:33.445126616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:37:33.447857162Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:37:34.178861743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:37:34.18800929Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:37:34.188038048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:34.253119796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:37:34.25628651Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:37:36.710895522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:37:36.71491855Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:37:36.808121812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:37:36.812551215Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:37:37.521758273Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-12T22:37:38.38537984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:37:38.388664833Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-12T22:37:39.187887085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:39.391152468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:37:39.39313617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:37:39.39709949Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:37:39.397103139Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:37:39.967999519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:37:39.971595587Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:37:39.971614758Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:39.971637782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:37:39.974582778Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:37:40.333893837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:37:40.337672852Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:37:40.33769138Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:40.337713817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:37:40.340614872Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:37:40.855174856Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
ts=2026-05-12T22:37:41.435231326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:37:42.693622857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:37:42.697788338Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:37:43.997656534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:37:44.005646428Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:37:44.18736788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:37:44.187405669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:37:44.187414906Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:37:44.18743055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:44.187469907Z caller=checkpoint.go:384 msg="flushing series" totalSize="316 kB" series=3
level=info ts=2026-05-12T22:37:44.192341975Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-12T22:37:44.193681923Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006244808s
level=debug ts=2026-05-12T22:37:46.723436673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:37:49.187820843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:51.839614953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:37:51.843872347Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:37:53.740630132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:37:53.74535226Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:37:54.187131935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:55.250792284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:37:55.254832074Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:37:55.992268599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:37:55.996008798Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:37:57.039925782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:37:57.043642978Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:37:57.043662732Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:37:57.043683314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:37:57.046587246Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:37:58.46586574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:37:58.469803263Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:37:58.62900976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:37:58.637003203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:37:59.18810282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:00.063482429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:38:00.067053584Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:38:04.187243553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:04.71239043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:38:04.716499159Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:38:06.754264985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:38:06.757778144Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:38:06.94034315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:38:06.947608005Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:38:08.467985802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:38:08.475617557Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:38:08.475639614Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:38:08.475677271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:38:08.483480855Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:38:09.188011997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:11.045856235Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:38:11.437290746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:38:13.786346677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:38:13.79360373Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:38:14.073302987Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:38:14.074369337Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:38:14.187151352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:38:14.18718074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:38:14.18718975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:38:14.187201048Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:38:14.187228263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:14.187392362Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-12T22:38:14.187401886Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:38:14.187419824Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.426µs success=true
level=debug ts=2026-05-12T22:38:14.187601375Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:38:14.289082389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:38:14.292370188Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:38:15.170026657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:38:15.173412639Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:38:17.521632893Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:38:19.187452641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:20.855134288Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-12T22:38:21.287349036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:38:21.295345696Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:38:21.633739691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:38:21.638019418Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:38:23.222513022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:38:23.230094425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:38:24.187052304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:24.187606751Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-12T22:38:25.736550516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:38:25.738390333Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:38:26.016197306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:38:26.025193235Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:38:26.597952237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:38:26.601893847Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:38:27.521816606Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:38:28.241583927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:38:28.245813637Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:38:29.187999986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:30.041995664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:38:30.045476212Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:38:30.046449437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:38:30.049373164Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:38:30.85442679Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-12T22:38:30.931780519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:38:30.93520985Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:38:34.187569192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:34.187680938Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:38:35.399547511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60260"
level=debug ts=2026-05-12T22:38:35.624500329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:38:36.32569857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:38:36.330007207Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:38:36.330043735Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:38:36.330097182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:38:36.333731282Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:38:37.521511134Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-12T22:38:37.713929127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:38:37.718122194Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:38:38.752754079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:38:38.756202084Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:38:39.187349573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:40.85859574Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-12T22:38:40.858662284Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=9
ts=2026-05-12T22:38:41.439268572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:38:43.453743763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:38:43.457481044Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:38:44.187755672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:38:44.187772216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:44.187795301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:38:44.18780487Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:38:44.188466253Z caller=checkpoint.go:372 msg="writing series" size="847 kB"
level=info ts=2026-05-12T22:38:44.374672752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:38:44.380083014Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:38:46.76371001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:38:46.767319018Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:38:47.03196504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:38:47.035560623Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:38:47.521227756Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:38:49.187669702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:50.854423391Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:38:50.854446898Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:38:52.302102974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:38:52.309777677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:38:52.452047135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:38:52.456307319Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:38:54.18712976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:54.188322883Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-12T22:38:56.030988999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:38:56.038533871Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:38:57.521393447Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:38:57.933522732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:38:57.938361717Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:38:59.187805567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:00.854853633Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T22:39:01.452428045Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:39:03.351935309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:39:03.358983331Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:39:03.520471553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:39:03.52767732Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:39:04.188578984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:04.188960019Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
ts=2026-05-12T22:39:05.40117605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37536"
level=debug ts=2026-05-12T22:39:07.521442607Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T22:39:07.76320756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:39:07.767175386Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:39:09.18730943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:39:10.79589711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:39:10.799503809Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:39:10.85461535Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-12T22:39:11.441174524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:39:12.208984538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:39:12.212510048Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:39:12.233125567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:39:12.237048125Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:39:12.608124604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:39:12.615931867Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:39:12.615956443Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:39:12.615986655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:39:12.62292227Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:39:14.07228892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:39:14.074366238Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:39:14.187408238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:39:14.187413027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:14.187437953Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:39:14.187446086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:39:14.189517051Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:39:14.189529427Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=7
ts=2026-05-12T22:39:15.073723293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48026"
level=info ts=2026-05-12T22:39:17.128116171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:39:17.133912493Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:39:17.521091185Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-12T22:39:17.668616551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:39:17.673240287Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:39:19.18698799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:20.86316932Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-12T22:39:20.863191778Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=info ts=2026-05-12T22:39:22.883606231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:39:22.888204916Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:39:24.187011329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:24.188398345Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-12T22:39:25.401106715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:39:25.404915883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:39:25.404934091Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:39:25.404953316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:39:25.408178801Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:39:25.759123164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:39:25.762662527Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:39:26.37451835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:39:26.378296959Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:39:26.529171484Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:39:26.778867028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:39:26.782564317Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:39:26.785902242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:39:27.521727927Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:39:27.521749271Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=warn ts=2026-05-12T22:39:29.187969811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:30.855116894Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:39:31.834943411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:39:31.842688719Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:39:32.270114712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:39:32.273787269Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:39:34.187832863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:34.188226434Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=info ts=2026-05-12T22:39:35.090355695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:35.097700486Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:35.09772171Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:39:35.097752751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:35.104958752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:39:35.899214438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:39:35.902931069Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:39:37.521756877Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:39:39.153780544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:39:39.161908213Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:39:39.187958994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:39:39.505594293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:39:39.51022525Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:39:40.855144766Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:39:41.444112627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:39:42.838564117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:39:42.842161556Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:39:42.99167517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:39:42.995339026Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:39:44.186980448Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:39:44.18701612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:39:44.187025316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:39:44.187033054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:44.188192632Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
ts=2026-05-12T22:39:45.075305307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58496"
level=debug ts=2026-05-12T22:39:47.521390092Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-12T22:39:47.521412211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-12T22:39:48.321843461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:48.330409395Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:39:49.188277926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:50.854720487Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-12T22:39:52.364277519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:39:52.371368159Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:39:53.969947316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:39:53.97388821Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:39:54.187293476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:54.188410763Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-12T22:39:54.991950122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36494"
level=info ts=2026-05-12T22:39:57.262885546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:39:57.266929801Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:39:57.417185603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:39:57.421704624Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:39:57.521116672Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-12T22:39:58.481708099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:39:59.187350943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:40:00.574696388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:40:00.583633098Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:40:00.855407799Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-12T22:40:04.187785499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:04.190741435Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-12T22:40:04.190756922Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=5
level=info ts=2026-05-12T22:40:04.830938719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:40:04.839484669Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:40:05.191953615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:40:05.195495163Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:40:05.955364603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:40:05.959042189Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:40:06.560801777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:40:06.564719288Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:40:07.039934618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:40:07.04425352Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:40:07.521735305Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T22:40:09.187191376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:40:10.270996319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:40:10.274591196Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:40:10.646387391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:10.650646306Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:10.650668012Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:40:10.650690737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:10.654301209Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:40:10.854960204Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:40:11.446983219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:40:11.916693533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:40:11.921340392Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:40:13.984904701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:40:13.988363136Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:40:14.07272223Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:40:14.074824525Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:40:14.187690686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:40:14.187717624Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:40:14.187726332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:40:14.187732238Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T22:40:14.187744944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:17.52150587Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-12T22:40:18.771587139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:40:19.187721774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:20.85485362Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=info ts=2026-05-12T22:40:21.478373752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:40:21.48260887Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:40:23.251328363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:40:23.255010102Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:40:23.484553704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:23.48853259Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:40:24.187244048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:24.188333039Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=info ts=2026-05-12T22:40:25.363328449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:40:25.368430035Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:40:26.615690976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:40:26.619844698Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:40:27.18364105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:40:27.192862437Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:40:27.52192539Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=warn ts=2026-05-12T22:40:29.187468258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:30.854694689Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-12T22:40:30.854726933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-12T22:40:31.101724956Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:40:32.013033285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:40:32.016502152Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:40:34.187842714Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-12T22:40:34.187852496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:40:36.045370995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:40:36.052811701Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:40:37.521450649Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=info ts=2026-05-12T22:40:37.890382406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:40:37.894598147Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:40:37.894625985Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:40:37.894655703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:40:37.898198827Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:40:39.187233412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:40.854584523Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=debug ts=2026-05-12T22:40:40.854607472Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:40:41.44934074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:40:42.020582792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:40:42.024361138Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:40:44.187472052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:40:44.18750455Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:40:44.18751317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:40:44.187516988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:44.187713768Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-05-12T22:40:45.077316861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42048"
level=info ts=2026-05-12T22:40:45.273186712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:40:45.280218382Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:40:46.793267697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:40:46.797159211Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:40:47.52135861Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=warn ts=2026-05-12T22:40:49.187113763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:50.855290704Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:40:51.089122346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:40:51.096793418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:40:53.536285539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:40:53.543359626Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:40:54.187914581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:54.188244128Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=debug ts=2026-05-12T22:40:54.188259755Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:40:54.78757034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:40:54.791675423Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:40:55.851029535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:40:55.854614927Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:40:56.151663713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:56.155723963Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:40:57.522644272Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:40:57.522668732Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-12T22:40:58.297512676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:40:58.300993755Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:40:58.301014127Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:40:58.301037955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:40:58.303887542Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=warn ts=2026-05-12T22:40:59.187994315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:00.331233104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:41:00.854362504Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:41:00.865618736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:41:00.869963427Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:41:02.015272878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:41:02.019355574Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:41:02.858151933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:41:02.862651611Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:41:02.86267846Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:41:02.862704888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:41:02.865726468Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:41:03.650727455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:41:03.654673415Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:41:04.187139638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:04.18833012Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:41:04.755597946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:41:04.759161149Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:41:07.521502658Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=warn ts=2026-05-12T22:41:09.187986993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:09.307636967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:41:09.311399239Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:41:09.842765771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:41:09.846156815Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:41:10.858202067Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:41:10.858233565Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-12T22:41:10.901049264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:41:11.450235903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:41:14.012494181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:41:14.019955749Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:41:14.072546243Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:41:14.074650395Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:41:14.187573771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:41:14.187603063Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:41:14.187610585Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:41:14.187621274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:14.18787741Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
ts=2026-05-12T22:41:15.078321333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52372"
level=info ts=2026-05-12T22:41:16.342088797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:41:16.346163911Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:41:16.880663292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:41:16.884036781Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:41:17.52097752Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:41:18.036378321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:41:18.039627379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=warn ts=2026-05-12T22:41:19.187981405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:20.855461547Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:41:20.855483727Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-12T22:41:22.649418524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:41:22.655334846Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:41:24.187034234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:24.235161952Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:41:24.235188405Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
level=info ts=2026-05-12T22:41:24.7473977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:41:24.750922587Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:41:26.684512062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:41:26.688171825Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:41:27.521783001Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-12T22:41:28.452098004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:41:28.460998598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:41:29.187961084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:30.855424668Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-12T22:41:31.616911401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:41:31.620933382Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:41:32.648848273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:41:32.656652404Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:41:32.656674017Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:41:32.656697955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:41:32.663857007Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:41:34.187821686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:34.18927629Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:41:34.189293708Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=debug ts=2026-05-12T22:41:35.613110825Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:41:37.521490308Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:41:37.660745031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:41:37.664802938Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:41:38.824012524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:41:38.828402492Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:41:39.187496455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:39.333105644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:39.339681553Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:39.339701486Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:41:39.339726867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:39.345825844Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:41:40.854500832Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
ts=2026-05-12T22:41:41.451546533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:41:42.660911644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:41:42.664513925Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:41:42.914120415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:41:42.92190566Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:41:43.00213097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:41:43.005466815Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:41:44.187628433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:41:44.187667449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:41:44.187679609Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:41:44.187686594Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-12T22:41:44.187701359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:41:45.079413988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42930"
level=info ts=2026-05-12T22:41:45.188996691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:41:45.19785933Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:41:45.211904157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:41:45.214997198Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:41:45.974315158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:41:45.977894816Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:41:47.521704967Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-12T22:41:49.187814936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:50.318804759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:41:50.322640476Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:41:50.467770698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:41:50.856600741Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-12T22:41:50.856623314Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=warn ts=2026-05-12T22:41:54.187369822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:54.188489142Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:41:54.29323412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:41:54.298250039Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:41:54.437653111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:41:54.445382934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:41:56.501747227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:56.508937068Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:41:57.521687206Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-12T22:41:59.1876461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:00.855041156Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:42:00.855063199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:42:03.682202319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:42:03.687050618Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:42:04.187688328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:04.187874861Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T22:42:05.594421081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:42:05.598023247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:42:07.521488043Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-12T22:42:07.79827522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:42:07.803218668Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:42:08.208467504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:42:08.212820572Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:42:08.969891342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:42:08.973806078Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:42:09.187186384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:09.978357671Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:42:09.978408467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:42:09.984909903Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:42:10.855756405Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:42:10.855780023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:42:11.452943099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:42:11.516927205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:42:11.521225636Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:42:12.86494864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:42:12.87293272Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:42:14.072335108Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:42:14.074463893Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:42:14.187269517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:14.187294712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:42:14.187317527Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:42:14.187326549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:42:14.188831353Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T22:42:17.521761525Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-12T22:42:19.188029038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:20.854438967Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-12T22:42:21.68601971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:42:21.691564157Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:42:21.738601619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:42:21.742043167Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:42:23.404382028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:42:23.407764715Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:42:23.601348618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:42:23.610024764Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:42:24.187211147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:24.188319124Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-12T22:42:24.599645646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:42:24.607395902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:42:26.666997036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:42:26.673808726Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:42:27.521606482Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=debug ts=2026-05-12T22:42:27.521628366Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-12T22:42:29.187766978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:30.187044341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:42:30.190782391Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:42:30.854406283Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:42:33.95763585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:42:33.96271765Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:42:34.187263488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:34.188419229Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-12T22:42:34.261540539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:42:34.265456502Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:42:34.554001614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:42:34.557234835Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:42:35.787491483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:42:35.790674299Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:42:37.12555217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:42:37.129509291Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:42:37.233360634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:42:37.234903363Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:42:37.521830066Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-12T22:42:38.393443123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:42:39.187832395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:40.855089368Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-12T22:42:40.903328301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:42:40.910849416Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:42:41.325043185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:42:41.332735675Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:42:41.454245442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:42:44.187585776Z caller=checkpoint.go:384 msg="flushing series" totalSize="392 kB" series=4
level=info ts=2026-05-12T22:42:44.187596279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:42:44.187619063Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:42:44.187627194Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:42:44.187610457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:44.195132211Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-12T22:42:44.196687373Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009133798s
ts=2026-05-12T22:42:45.082458788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56728"
level=warn ts=2026-05-12T22:42:49.18734239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:51.619282645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:51.634798035Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:51.634820314Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:51.634844573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:51.638690264Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:42:53.595277934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:42:53.60039734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:42:54.140597778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:42:54.145347927Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:42:54.145367668Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:54.145389962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:42:54.149017501Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:42:54.187274842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:56.561551127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:42:56.569194111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:42:56.596266187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:42:56.599534669Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:42:57.102195798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:42:57.105869897Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:42:58.798674994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:42:58.804534785Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:42:59.18802138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:00.02430704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:43:00.028475551Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:43:00.383967286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:43:00.387705013Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:43:03.18301653Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:43:04.187763612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:43:05.415701371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55630"
level=info ts=2026-05-12T22:43:08.643488407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:43:08.651160122Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:43:08.708458904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:43:08.712504047Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:43:09.187765351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:11.359979319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:43:11.363512545Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:43:11.455837664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:43:13.283477216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:43:13.287544295Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:43:13.344590048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:43:13.348794282Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:43:14.037233968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:43:14.040587522Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:43:14.072120248Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:43:14.074225289Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:43:14.186970883Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:43:14.187043893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:43:14.187064507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:14.187091499Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:43:14.187100027Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:43:14.187152928Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-12T22:43:14.187160672Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:43:14.187178158Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.719µs success=true
level=debug ts=2026-05-12T22:43:14.187349711Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:43:14.290595331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:43:14.294089446Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:43:15.83296191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:43:15.836387987Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:43:16.417340912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:43:17.521013019Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:43:17.747538091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:43:17.751351023Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:43:19.187807286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:19.401120165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:43:19.404679968Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:43:20.855151499Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-12T22:43:21.706684167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:43:21.717242931Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:43:24.18793125Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-12T22:43:24.18793116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:25.546711298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:43:25.55053359Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:43:27.521615999Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-12T22:43:29.187118953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:29.750196628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:43:29.75604869Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:43:30.855095823Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-12T22:43:34.187836751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:34.1878939Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-12T22:43:36.052498869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:43:36.059888582Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:43:37.521336387Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-12T22:43:38.899386345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:43:38.899449102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="176 kB" avg_comp="176 kB" total_uncomp="906 kB" avg_uncomp="906 kB" idle=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:43:38.903850151Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=warn ts=2026-05-12T22:43:39.187982886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:39.402273799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:43:39.407397541Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:43:39.541018871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:43:39.548461311Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:43:39.585721812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:43:39.5900577Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:43:40.857348154Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:43:40.857371643Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=9
ts=2026-05-12T22:43:41.45716512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:43:42.006384928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:43:42.00980057Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:43:42.009819388Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:43:42.009840666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:43:42.012755359Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:43:42.442119833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:43:42.445936253Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:43:42.445955631Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:43:42.445976304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:43:42.449130102Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:43:44.186936035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:43:44.186967794Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:43:44.18697642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:43:44.187010144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:44.188557579Z caller=checkpoint.go:372 msg="writing series" size="855 kB"
level=info ts=2026-05-12T22:43:45.122726197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:43:45.132090637Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:43:47.521221429Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:43:49.187364212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:49.370411627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:43:49.373923454Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:43:49.373942173Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:43:49.373961456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:43:49.376926431Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:43:49.535513646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:43:49.539599902Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:43:50.854516856Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:43:50.854541512Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:43:52.176498544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:43:52.184991841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:43:52.827015582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:43:52.831194603Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:43:54.18699871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:54.188180557Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-12T22:43:54.669769392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:43:54.673579871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:43:54.67359754Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:43:54.673618118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:43:54.676497598Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-12T22:43:55.016427001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34436"
level=info ts=2026-05-12T22:43:55.408972867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:43:55.41281386Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=debug ts=2026-05-12T22:43:57.52075356Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:43:58.058753239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:43:58.063083846Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:43:58.130320782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:43:58.136263346Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:43:58.330978993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:43:58.33435237Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:43:59.18761551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:00.854424402Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:44:01.131517344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:44:01.135195995Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:44:01.135216663Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:44:01.135238351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:44:01.137959474Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:44:01.246186529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:44:01.24965451Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:44:02.350932626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:44:02.354434985Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:44:04.187048837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:04.188494154Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
ts=2026-05-12T22:44:05.420506217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57244"
level=info ts=2026-05-12T22:44:06.183427782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:44:06.187898597Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:44:07.305998853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:44:07.315952402Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:44:07.521532742Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T22:44:09.187794545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:10.048817743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:44:10.052616615Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:44:10.657326457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:44:10.660848331Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:44:10.660867307Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:44:10.660886944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:44:10.664056662Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:44:10.734182259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:10.737743367Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:10.737760971Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=debug ts=2026-05-12T22:44:10.855041308Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-12T22:44:10.961452353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:10.965054719Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-12T22:44:11.458661538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:44:12.382729512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:44:12.749669101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:44:12.753989478Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:44:13.208449775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:44:13.212140745Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:44:13.50497981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:44:13.508847134Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:44:14.072667469Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:44:14.074756136Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:44:14.187679854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:14.187712995Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:44:14.18773958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:44:14.187750229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:44:14.188314337Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:44:14.18834074Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=debug ts=2026-05-12T22:44:17.521825722Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-05-12T22:44:17.957898219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:44:18.236800261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:44:18.24068507Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=warn ts=2026-05-12T22:44:19.187729348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:19.816513087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:44:19.824277843Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:44:20.868900921Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-12T22:44:20.86892556Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=info ts=2026-05-12T22:44:21.862207991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:21.865523635Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:44:23.035347989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:44:23.039377292Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:44:24.187283903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:24.187481692Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=info ts=2026-05-12T22:44:25.281675233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:44:25.285802027Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:44:26.216285941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:44:26.220215269Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:44:26.256330727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:44:26.264303404Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:44:26.264324891Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:44:26.264362771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:44:26.271362579Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:44:27.151782612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:44:27.155102198Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:44:27.522384817Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:44:27.522413771Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-12T22:44:28.806886939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:44:28.810537711Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:44:29.087600751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:44:29.090796035Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:44:29.187843304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:29.266499744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:44:29.273461548Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:44:30.85423526Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:44:32.531121402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:44:32.535950167Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:44:34.187385415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:34.187748179Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
ts=2026-05-12T22:44:35.421701421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58030"
level=info ts=2026-05-12T22:44:35.911570004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:44:35.915026853Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:44:37.103538361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:44:37.106862753Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:44:37.521264982Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:44:37.578338168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:44:37.583619011Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:44:38.973602402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:44:38.977242431Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:44:39.187410013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:40.854327798Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:44:41.460281242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:44:42.009216252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:44:42.016007044Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:44:42.730140426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:42.733500498Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:44:44.187291746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:44:44.187317828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:44:44.187308687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:44.187326065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:44:44.187406946Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=debug ts=2026-05-12T22:44:45.957020166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:44:46.111702221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:44:46.11890584Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:44:46.381703597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:44:46.388810546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:44:47.521021553Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-12T22:44:47.521043997Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-12T22:44:49.187488933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:50.854381835Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-12T22:44:54.187570015Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-12T22:44:54.187567923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:44:55.02275559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34400"
level=debug ts=2026-05-12T22:44:57.521250612Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-12T22:44:58.660194005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:44:58.665096846Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:44:58.665118584Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:44:58.665141126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:44:58.668379696Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:44:59.187690249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:00.399964202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:45:00.408453926Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:45:00.50005638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:45:00.855114919Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-12T22:45:04.184838356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:45:04.187258016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:45:04.18903482Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:45:04.190459899Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-12T22:45:04.190473232Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=5
ts=2026-05-12T22:45:05.423226073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47340"
level=info ts=2026-05-12T22:45:05.468121606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:45:05.471521522Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:45:06.226556093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:45:06.229862446Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:45:06.230847681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:45:06.233618424Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:45:06.263844088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:45:06.271216542Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:45:07.008190242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:45:07.011481946Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:45:07.520864665Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-12T22:45:09.187452098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:09.362296989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:45:09.367321468Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:45:10.535995465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:10.544139425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:45:10.854948587Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:45:11.462981588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:45:12.59347535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:45:12.597862243Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:45:14.073030799Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:45:14.074101952Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:45:14.075385518Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:45:14.07542821Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976250 n=1
level=debug ts=2026-05-12T22:45:14.075469321Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976250 n=1
level=info ts=2026-05-12T22:45:14.18796034Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:45:14.187988614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:45:14.187997993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:45:14.188001861Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T22:45:14.188005892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:15.479976062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:45:15.488554176Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:45:16.335503796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:45:16.33921229Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:45:17.263022166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:45:17.521806623Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-12T22:45:18.071437017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:45:18.076147554Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:45:19.187608395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:19.490113206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:45:19.493591642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:45:20.397910163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:45:20.401355871Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:45:20.544426167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:45:20.548498343Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:45:20.854525058Z caller=checkpoint.go:372 msg="writing series" size="487 kB"
level=info ts=2026-05-12T22:45:21.373185218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:45:21.376625845Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:45:24.187991865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:24.188001683Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=info ts=2026-05-12T22:45:25.825655211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:45:25.830523463Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:45:27.20746796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:45:27.211387283Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:45:27.211406474Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:45:27.211429449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:45:27.214961635Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:45:27.520941272Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=warn ts=2026-05-12T22:45:29.187734714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:29.387964984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:45:29.395443086Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:45:30.854633841Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-12T22:45:30.854655299Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-12T22:45:32.789892177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:45:32.79480167Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:45:33.891928803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:45:33.897539969Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:45:33.897563939Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:45:33.897585092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:45:33.900726711Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:45:34.187896601Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-12T22:45:34.187892307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:45:35.42429448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46966"
level=debug ts=2026-05-12T22:45:37.521352707Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=debug ts=2026-05-12T22:45:37.808086872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:45:38.178344995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:45:38.185530538Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:45:39.187376365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:40.855417673Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=debug ts=2026-05-12T22:45:40.855444362Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:45:41.46625108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:45:44.187311224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:45:44.187336324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:44.187360082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:45:44.187371556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:45:44.187430485Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-12T22:45:45.092906861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58764"
level=info ts=2026-05-12T22:45:46.102102582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:45:46.106048724Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:45:47.5208897Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-12T22:45:47.570121218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:45:47.574488216Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-12T22:45:49.187485473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:50.61529585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:45:50.623423769Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:45:50.623446263Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:45:50.623469539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:45:50.63069184Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:45:50.854583148Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:45:51.191567732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:45:51.195993959Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:45:52.501630756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:45:52.50557654Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:45:53.047153148Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:45:53.319742446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:45:53.32296392Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:45:53.532502302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:45:53.539015274Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:45:54.187579637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:54.187765137Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-05-12T22:45:54.706640228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:54.713608722Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:54.713629722Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:45:54.713654899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:54.720219262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:45:56.284894087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:45:56.288695124Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:45:56.858039972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:45:56.861733153Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:45:57.523182093Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:45:57.523205561Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=5
level=info ts=2026-05-12T22:45:59.077813875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:45:59.081721044Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=warn ts=2026-05-12T22:45:59.187978942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:00.725192532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:46:00.73379271Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:46:00.855080422Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:46:02.198025688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:46:02.201946243Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:46:02.616991048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:46:02.621282418Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:46:03.831555946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:46:03.835603736Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:46:04.187728989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:04.187812924Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-12T22:46:06.161142247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:46:06.164469081Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:46:07.521486783Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=warn ts=2026-05-12T22:46:09.187701385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:09.231875741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:46:09.240552778Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:46:09.807999376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:46:09.811511709Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:46:10.854415585Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:46:10.854439444Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-12T22:46:10.896727539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:46:10.9001006Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-12T22:46:11.46814212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:46:11.490880214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:46:11.498054252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:46:11.983382244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:46:11.987628565Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:46:13.184045858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:46:13.188133369Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:46:14.072984426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:46:14.074051108Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:46:14.187816317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:46:14.187844089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:46:14.187852804Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:46:14.187883706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:14.188190995Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:46:15.639945686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:46:15.644127959Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:46:17.521122786Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:46:18.446841086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:46:18.450493327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:46:19.187696438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:20.855061614Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:46:20.855104529Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-12T22:46:20.913311936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:46:20.914971416Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:46:22.56558152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:46:22.569620447Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:46:23.593513487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:46:24.187720909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:24.235016742Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:46:24.235045502Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
level=info ts=2026-05-12T22:46:24.364806507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:46:24.37160375Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-12T22:46:25.032157599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49794"
level=info ts=2026-05-12T22:46:25.841751259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:46:25.849579534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:46:27.521722138Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-12T22:46:29.18740216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:30.855121458Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=warn ts=2026-05-12T22:46:34.188053315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:34.189453483Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:46:34.189474548Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-12T22:46:34.481924612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:46:34.488604784Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:46:36.731544077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:46:36.739375767Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:46:37.521664111Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:46:37.676183247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:46:37.683984169Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:46:37.794188519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:46:37.797715537Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:46:39.107305283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:46:39.11769546Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:46:39.187943772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:40.855043122Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=info ts=2026-05-12T22:46:40.943415027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:46:40.947689698Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-12T22:46:41.469774254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:46:41.779210524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:46:41.783483754Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:46:42.584637515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:46:42.588329864Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:46:43.974274883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:46:43.978152772Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:46:44.187641865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:46:44.187647897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:44.187669603Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:46:44.187678028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:46:44.187709145Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-12T22:46:47.5216456Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-12T22:46:48.812909314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:46:49.187865013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:50.086826766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:46:50.091893044Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:46:50.857136465Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:46:50.857159489Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=5
level=info ts=2026-05-12T22:46:53.342146039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:46:53.350844492Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:46:54.187029244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:54.188155469Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-12T22:46:55.034940743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34484"
level=info ts=2026-05-12T22:46:55.341022572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:46:55.34454873Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:46:55.662421235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:46:55.66623306Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:46:56.702543674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:46:57.312194022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:46:57.31628547Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:46:57.520935665Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-12T22:46:59.187533432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:59.880399579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:46:59.884783866Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T22:46:59.884810527Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:46:59.884835851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:46:59.888720106Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:47:00.140176346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:47:00.143643478Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:47:00.73712338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:47:00.74163473Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:47:00.855800055Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:47:00.855821059Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:47:02.940010302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:47:02.944185801Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:47:03.899769452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:47:03.903232732Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:47:04.187307547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:04.187378353Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-05-12T22:47:05.432037121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40496"
level=debug ts=2026-05-12T22:47:07.521728231Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=info ts=2026-05-12T22:47:09.106806142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:47:09.114195403Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:47:09.187407659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:10.855438118Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:47:10.855465893Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-12T22:47:11.472020345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:47:14.06875046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:47:14.071860416Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=error ts=2026-05-12T22:47:14.072692468Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:47:14.074695654Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:47:14.187670219Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:47:14.187703062Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:47:14.187712246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:14.18772104Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:47:14.188101103Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=info ts=2026-05-12T22:47:17.036956076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:47:17.044237585Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:47:17.099673612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:47:17.10422977Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:47:17.521383547Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-12T22:47:19.188008668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:19.383187355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:47:19.391964047Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:47:20.854585817Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-12T22:47:21.157575408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:47:21.161264871Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:47:21.378849796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:47:21.384707289Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:47:24.18761008Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-05-12T22:47:24.187636905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:24.71545495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:47:25.038592882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53748"
level=info ts=2026-05-12T22:47:25.541481975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:47:25.549435434Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:47:26.401972211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:47:26.40550059Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:47:26.405521688Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:47:26.40554351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:47:26.408585976Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:47:27.521973951Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
level=debug ts=2026-05-12T22:47:27.521996601Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-12T22:47:28.294866851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:47:28.298338712Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:47:29.187719256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:29.868699435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:47:29.872390023Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:47:29.872407628Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:47:29.872428039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:47:29.875869418Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:47:30.854792196Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:47:31.828538583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:47:31.832021036Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:47:32.18260379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:47:33.931398656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:47:33.937895769Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:47:34.187284174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:34.187370407Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-12T22:47:34.354614279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:47:34.358197016Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
ts=2026-05-12T22:47:35.433726028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41934"
level=info ts=2026-05-12T22:47:36.246784721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:47:36.250784191Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:47:36.250808607Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:47:36.250833818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:47:36.254306518Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:47:37.108662599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:47:37.11261239Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:47:37.52113593Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-12T22:47:38.422190836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:47:38.425998182Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:47:39.187636466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:40.568948391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:47:40.573024557Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:47:40.855040515Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
ts=2026-05-12T22:47:41.474026617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:47:43.885757721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:47:43.893777348Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:47:44.187206814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:47:44.187248203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:47:44.187257216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:47:44.187282656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:44.187314934Z caller=checkpoint.go:384 msg="flushing series" totalSize="405 kB" series=4
level=info ts=2026-05-12T22:47:44.192820101Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-12T22:47:44.194002529Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006696778s
level=info ts=2026-05-12T22:47:44.282175747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:47:44.285379988Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:47:49.187300317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:49.541930119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:47:49.545733939Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:47:52.669187449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:47:52.672668332Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:47:54.187762246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:54.43518564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:47:54.438668419Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:47:55.614709597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:47:55.618786572Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:47:56.012917412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:47:56.016391494Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:47:57.0392228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:47:57.042832487Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:47:58.303638872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:47:58.307722175Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:47:59.187352921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:59.436829596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:47:59.436861598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:47:59.440262247Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:48:00.488860712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:48:00.493616001Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:48:01.165521429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:48:01.173349576Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:48:01.173370902Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:01.173396837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:48:01.180264239Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:48:04.187311354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:04.261491167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:48:04.265912498Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:48:07.356213504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:48:07.359659Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:48:08.267243752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:48:09.187576096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:10.030270384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:48:10.039160139Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-12T22:48:11.476166543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:48:12.455400725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:48:12.459223549Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T22:48:14.07290003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:48:14.073951092Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:48:14.187820922Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-12T22:48:14.187830255Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-12T22:48:14.187850379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:48:14.187859102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:48:14.187883836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:14.187977959Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-12T22:48:14.187986084Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:48:14.188003289Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.497µs success=true
level=debug ts=2026-05-12T22:48:14.188175659Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:48:14.199034176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:48:14.206504796Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:48:14.921805055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:48:14.925211391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:48:15.068953769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:48:15.078964239Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-oref\"}"
ts=2026-05-12T22:48:15.102725076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54848"
level=debug ts=2026-05-12T22:48:17.522503685Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:48:19.187577267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:19.711955256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:48:19.716022874Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:48:19.978310806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:48:19.981584693Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:48:20.855137501Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-12T22:48:21.576374026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:48:21.583775337Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:48:21.834389522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:48:21.838028713Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:48:23.122754374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:48:23.12636452Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=warn ts=2026-05-12T22:48:24.187666536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:24.188897789Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-12T22:48:25.045593855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:37964"
level=info ts=2026-05-12T22:48:25.724682428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:48:25.728256545Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:48:26.084387415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:26.090859211Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:26.090882074Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:26.090904182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:26.097876359Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:48:27.274359719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:48:27.278243865Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-12T22:48:27.522031302Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:48:28.967850734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:48:28.974885757Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:48:29.187455852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:30.164104457Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:48:30.164120612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="43 kB" avg_uncomp="43 kB" idle=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:48:30.167671428Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:48:30.855531657Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-12T22:48:33.075115849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:48:33.078631852Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T22:48:34.187164246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:34.188351336Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
ts=2026-05-12T22:48:35.438249807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44970"
level=info ts=2026-05-12T22:48:36.685122242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:48:36.688586209Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:48:37.508891622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:48:37.515497839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:48:37.522684582Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-12T22:48:37.667059899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:48:37.670280105Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:48:39.187687797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:40.021757678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:48:40.025119042Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:48:40.427501885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:48:40.433280248Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:48:40.857042032Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-12T22:48:40.85706411Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=9
ts=2026-05-12T22:48:41.479782925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:48:41.715833007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:48:41.719577309Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=info ts=2026-05-12T22:48:42.147783574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:42.154360754Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:48:44.187596071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:48:44.187628314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:48:44.187636998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:48:44.187629245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:44.189038992Z caller=checkpoint.go:372 msg="writing series" size="859 kB"
level=info ts=2026-05-12T22:48:46.561807852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:48:46.565576991Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:48:47.522402813Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T22:48:48.059841443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:48:48.063886174Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:48:49.187527029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:50.783371081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:48:50.787603125Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:48:50.855787015Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:48:50.855812219Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:48:51.448662606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:48:51.454760454Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:48:51.45478012Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:51.454801747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:48:51.460361106Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:48:52.046944606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:48:52.050552129Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:48:54.187612858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:54.188751743Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-12T22:48:54.452091763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:48:54.456463382Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-12T22:48:55.048815383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45132"
level=debug ts=2026-05-12T22:48:57.522192816Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:48:59.071836708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:48:59.075412968Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-12T22:48:59.187832401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:00.855266899Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T22:49:03.751638195Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:49:03.972365444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:49:03.980146737Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:49:04.187626298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:04.188897074Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
ts=2026-05-12T22:49:05.439376606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54368"
level=info ts=2026-05-12T22:49:06.042436572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:06.048752647Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:49:07.522382744Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T22:49:07.951584502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:49:07.954890192Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:49:09.187675254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:49:09.709603315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:49:09.713508181Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:49:10.025429602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:49:10.325850173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:49:10.3328421Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:49:10.855157078Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-12T22:49:11.173807332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:49:11.179464082Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-12T22:49:11.482507704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:49:12.107652245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:49:12.111432573Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:49:12.578691561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" idle=1 full=1 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:49:12.586941686Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:49:14.072727313Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:49:14.073854001Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:49:14.18763052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:49:14.18763083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:14.187656021Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:49:14.187667051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:49:14.189310014Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-12T22:49:14.189322887Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=7
level=info ts=2026-05-12T22:49:14.468517644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:49:14.472421697Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-12T22:49:15.107359616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58630"
level=debug ts=2026-05-12T22:49:17.521987708Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-12T22:49:17.672494418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:49:17.676750078Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=warn ts=2026-05-12T22:49:19.187824749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:49:19.657315642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:49:19.661042632Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:49:20.867302551Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-12T22:49:20.867327342Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=info ts=2026-05-12T22:49:21.354835608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:49:21.358614605Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:49:22.915385282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:49:22.923212308Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:49:24.076825203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:49:24.084415511Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:49:24.18785323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:24.189018201Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-12T22:49:27.342188397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:49:27.346361984Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:49:27.523287308Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:49:27.523313728Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-12T22:49:28.404322987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:49:28.408763328Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:49:29.187192223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:30.855224549Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:49:33.003350228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:49:33.00735866Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:49:33.031422167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:49:33.034955305Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:49:34.187811821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:34.189211608Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=info ts=2026-05-12T22:49:34.654767121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:49:34.659089219Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:49:34.659110186Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:34.659134611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:49:34.662730364Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:49:35.586127475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:49:35.590137657Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:49:35.762596412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:49:35.766196042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:49:37.522273002Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:49:38.739686483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:38.747765917Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:49:39.187638194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:39.195920699Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:49:40.856021046Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:49:41.485235965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:49:42.069506244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:49:42.07467895Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:49:44.187405216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:49:44.187405159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:44.187436094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:49:44.187444897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:49:44.188692359Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-12T22:49:44.310453588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:44.314983467Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:44.315003047Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:44.315026251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:44.318524308Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:49:44.692942057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:49:44.69650702Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:49:45.108999622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56646"
level=info ts=2026-05-12T22:49:45.355653332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:49:45.359741176Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:49:46.108397179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:49:46.109911807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:49:46.156103434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:49:46.159134269Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:49:47.521710869Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T22:49:49.187041212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:50.855033101Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-05-12T22:49:50.855055218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-12T22:49:52.063499935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:49:52.070646735Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:49:53.536707629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:49:53.544306547Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:49:54.187303608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:54.188427544Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:49:54.566913354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:54.571139102Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:49:54.961726174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:49:54.965457907Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:49:54.965477777Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:54.965500837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:49:54.96867164Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-12T22:49:55.055117333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39746"
level=debug ts=2026-05-12T22:49:57.522127017Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T22:49:59.18781234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:00.855803413Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=info ts=2026-05-12T22:50:00.987099634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:50:00.991235594Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:50:02.33179629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:50:02.336335673Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:50:02.389478194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:50:02.393129117Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:50:02.393150542Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:02.39317262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:50:02.396432416Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:50:02.797510131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:50:02.804598643Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:50:04.187084201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:04.191016285Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-12T22:50:04.191032783Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=4
level=debug ts=2026-05-12T22:50:05.287250551Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:50:07.522486606Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-12T22:50:07.917178106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:50:07.92172014Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:50:08.244569494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:50:08.248880736Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:50:09.187980199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:50:09.20211199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:50:09.206207543Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:50:10.855108714Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:50:11.486667009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:50:11.843088809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:50:11.846593334Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:50:13.986199073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:50:13.990557033Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:50:13.990582388Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:13.990607333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:50:13.993960964Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=debug ts=2026-05-12T22:50:14.072891758Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:50:14.073950199Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:50:14.187360458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:50:14.187388301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:50:14.187399045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:50:14.187407086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:14.188516938Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-12T22:50:15.110449151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41564"
level=info ts=2026-05-12T22:50:15.647397347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:50:15.651377255Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:50:16.222975792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:50:16.226607621Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:50:16.328948733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:50:16.333159606Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:50:17.304666538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:50:17.311979252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:50:17.473515916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:50:17.477707621Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:50:17.521900956Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-12T22:50:17.768526932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:50:17.775355558Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:50:19.18740845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:20.856238839Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=info ts=2026-05-12T22:50:20.95719439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:50:20.960770984Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=warn ts=2026-05-12T22:50:24.187519023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:24.188638836Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=debug ts=2026-05-12T22:50:24.344325324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:50:24.776790601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:50:24.780756811Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:50:26.046760881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:50:26.050525189Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:50:27.397344677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:50:27.401289716Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:50:27.401322302Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:27.401356616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:50:27.404470137Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:50:27.52192881Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=info ts=2026-05-12T22:50:27.941119215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:50:27.944642665Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T22:50:29.18753906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:50:29.916455337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:50:29.919766175Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:50:29.919784053Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:29.919804493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:50:29.922662597Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:50:30.059961561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:50:30.063618304Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:50:30.063640514Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:30.220873209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:50:30.224220148Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:50:30.85502841Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-12T22:50:30.855051374Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-12T22:50:32.366797728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:50:32.370410406Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:50:32.370429915Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:32.370451123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:50:32.373336777Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:50:33.082269836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:50:33.085893265Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:50:34.187372137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:34.188518501Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-12T22:50:36.285819599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:50:36.293392373Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:50:37.522208107Z caller=checkpoint.go:372 msg="writing series" size="558 kB"
level=info ts=2026-05-12T22:50:37.993990729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:50:38.002421638Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:50:39.187721206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:50:39.344296014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:50:39.354225516Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:50:40.856371378Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=debug ts=2026-05-12T22:50:40.856397007Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:50:40.882250947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:50:40.885604123Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:50:41.105377072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:50:41.487631666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:50:44.171325147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:50:44.178110792Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:50:44.187115162Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:50:44.187130035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:50:44.187137295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:50:44.187158202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:44.188402434Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-12T22:50:44.782913088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:50:44.786491696Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:50:45.11157183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50400"
level=info ts=2026-05-12T22:50:46.285116624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:50:46.288903251Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:50:47.522290259Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-12T22:50:47.62858672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:50:47.63298179Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:50:49.187290263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:50:50.432389413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:50:50.437751745Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:50:50.855054899Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:50:51.793354389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:50:51.797229166Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:50:54.081878733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:50:54.085571246Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:50:54.085595967Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:54.085627415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:50:54.08867164Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:50:54.18711126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:54.189469367Z caller=checkpoint.go:372 msg="writing series" size="368 kB"
level=debug ts=2026-05-12T22:50:54.189490875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:50:54.592247112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:50:54.596112197Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:50:57.30612553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:50:57.310442639Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:50:57.525133157Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:50:57.525157147Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=warn ts=2026-05-12T22:50:59.187292302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:50:59.31914061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:50:59.326207441Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:50:59.326231136Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:50:59.326269072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:50:59.332886899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:51:00.337910289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:51:00.341579124Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:51:00.854929171Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:51:02.33412377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:51:02.34109778Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:51:02.535637337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:51:02.539641369Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:51:02.758130417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:51:02.762334968Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:51:04.187285977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:04.188622415Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-12T22:51:05.447002306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:51:05.450393499Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:51:05.804477929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:51:05.808428038Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:51:07.262585948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:51:07.267090283Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:51:07.521941738Z caller=checkpoint.go:372 msg="writing series" size="504 kB"
level=info ts=2026-05-12T22:51:08.414262812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:51:08.417907922Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-12T22:51:09.187863347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:09.531618409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:51:09.539127987Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:51:10.015772381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:51:10.019337193Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:51:10.855989066Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=info ts=2026-05-12T22:51:11.285345389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:51:11.288985238Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-12T22:51:11.488786675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:51:12.384258545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:51:12.387750728Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:51:14.072327752Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:51:14.074423426Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:51:14.187214761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:51:14.187250258Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:51:14.187260336Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:51:14.187252615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:14.188697115Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-05-12T22:51:14.188707995Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-12T22:51:14.642267202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="188 kB" avg_uncomp="188 kB" idle=1 labels="{stream=\"build-e2e-hvns\"}"
level=debug ts=2026-05-12T22:51:14.642386888Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-12T22:51:14.646697361Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=info ts=2026-05-12T22:51:14.740440866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:51:14.743853791Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:51:15.4016311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:51:15.405757877Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:51:16.749223482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:51:16.756153509Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:51:17.521867641Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:51:17.720678388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:51:17.725194598Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:51:19.187327592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:19.51906512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:51:20.855668051Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-05-12T22:51:24.187329296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:24.241911897Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:51:24.241938055Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=info ts=2026-05-12T22:51:27.372510064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:51:27.377057977Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:51:27.522636254Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-12T22:51:28.108291859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:51:28.117481754Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=warn ts=2026-05-12T22:51:29.187047648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:29.808846457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:51:29.812818388Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=debug ts=2026-05-12T22:51:30.855141675Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-05-12T22:51:34.187930044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:34.191620524Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:51:34.191643993Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-12T22:51:34.80371464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:51:34.810348614Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-12T22:51:35.447550192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44326"
level=info ts=2026-05-12T22:51:36.773528396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:51:36.782259122Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:51:37.522510561Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T22:51:39.18757084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:40.246615002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:51:40.252881434Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:51:40.855620223Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
ts=2026-05-12T22:51:41.489942636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:51:41.845928716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:51:41.849521474Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:51:42.416301011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:51:42.420126982Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:51:42.554528633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:51:42.559235856Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:51:43.935898739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:51:43.939556671Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=info ts=2026-05-12T22:51:44.187488824Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:51:44.187520027Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:51:44.187529312Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:51:44.187543529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:44.188771864Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:51:44.2939585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:51:44.302349536Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T22:51:45.116509585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40926"
level=info ts=2026-05-12T22:51:45.49499021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:51:45.499557736Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=debug ts=2026-05-12T22:51:47.522026529Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-12T22:51:47.806813329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:51:47.810352942Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:51:48.047058618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:51:48.047123248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="185 kB" avg_uncomp="185 kB" idle=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:51:48.050949867Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=warn ts=2026-05-12T22:51:49.187428468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:50.496100564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:51:50.499795751Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:51:50.499814745Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:51:50.499834861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:51:50.502692898Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:51:50.859519245Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:51:50.859553823Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=5
level=info ts=2026-05-12T22:51:51.664986568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:51:51.668543691Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:51:52.657190132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:51:52.663746943Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:51:53.810889288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:51:53.815463337Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:51:54.154829363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:51:54.159209129Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:51:54.187561068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:54.18862477Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:51:54.713145727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:51:54.716688837Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:51:56.264335291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:51:56.267691439Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:51:56.653411155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:51:56.65741874Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:51:56.657444693Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:51:56.657467405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:51:56.661336571Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:51:57.522662054Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-12T22:51:58.155365195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:51:59.187578761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:00.014919926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:52:00.0183285Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:52:00.85550038Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:52:00.855522758Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:52:01.555977482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:52:01.563570479Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:52:02.258384419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:52:02.261741978Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:52:04.187970035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:04.189094208Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T22:52:06.071197096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:52:06.074716193Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:52:06.083874938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:52:06.08689164Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:52:07.52178385Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=info ts=2026-05-12T22:52:07.725243276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:52:07.728603699Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=warn ts=2026-05-12T22:52:09.187318464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:09.330935807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:52:09.334333955Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=debug ts=2026-05-12T22:52:10.855703549Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:52:10.855724746Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:52:11.452556194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:52:11.456632416Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-12T22:52:11.492325982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:52:11.516023958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:52:11.519305552Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:52:11.519324017Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:52:11.519344844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:52:11.522254286Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:52:14.073005265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:52:14.074052739Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:52:14.187821844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:52:14.187849408Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:52:14.187858249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:52:14.187907364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:14.189319371Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-12T22:52:16.478026894Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:52:16.547080297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:52:16.551961268Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:52:17.234980122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:52:17.243245209Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:52:17.243268806Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:52:17.24329184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:52:17.251344195Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:52:17.522250003Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-12T22:52:19.187394983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:20.85570713Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-12T22:52:23.847936807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:52:23.854942497Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:52:24.187680115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:24.188820748Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-12T22:52:27.166516532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:52:27.1705462Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:52:27.522949404Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=debug ts=2026-05-12T22:52:27.52297101Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-12T22:52:27.681083482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:52:27.689962915Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:52:27.689983164Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:52:27.690005369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:52:27.697974286Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:52:27.981750254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:52:27.985898556Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=info ts=2026-05-12T22:52:28.500386243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:52:28.504385883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:52:29.18742763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:29.713854311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:52:29.719399977Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:52:30.710895889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:52:30.714917876Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=debug ts=2026-05-12T22:52:30.855221281Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:52:31.935512088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:52:31.939769847Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:52:32.796751967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:52:32.801418057Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:52:34.187126313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:34.188259738Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
ts=2026-05-12T22:52:35.450189785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41030"
level=info ts=2026-05-12T22:52:36.744425716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:52:36.748933673Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:52:36.975687586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:52:36.983365999Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:52:37.218106573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:52:37.221482135Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:52:37.522273396Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T22:52:39.18717192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:39.668609797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:52:39.670836995Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:52:39.672328159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:52:39.680372436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:52:39.687713522Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:52:40.855324521Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
ts=2026-05-12T22:52:41.493711956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:52:41.591647063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:52:41.601476561Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:52:42.031587527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:52:42.035300879Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:52:43.625093514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:52:43.628710391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:52:43.91462611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:52:43.918870966Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:52:44.187457443Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:52:44.187495869Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:52:44.18750456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:52:44.187546154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:44.188562501Z caller=checkpoint.go:384 msg="flushing series" totalSize="193 kB" series=4
level=info ts=2026-05-12T22:52:44.193359414Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-12T22:52:44.194572007Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006437588s
level=info ts=2026-05-12T22:52:44.908955669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:52:44.916885511Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:52:45.119758771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46274"
level=debug ts=2026-05-12T22:52:48.348689928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:52:48.623373442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:52:48.628406328Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:52:49.18810061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:49.485566031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:52:49.489766633Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:52:51.059963916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:52:51.063713705Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:52:52.147360334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:52:52.151278034Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:52:52.5344501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:52:52.541941463Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:52:52.679541021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:52:52.684169957Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:52:53.214482984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:52:53.21801536Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=warn ts=2026-05-12T22:52:54.187955717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:55.980382562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:52:55.985806299Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:52:59.187185473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:59.644754466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:52:59.648377766Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:52:59.849496553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:52:59.853270317Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-12T22:53:04.187830048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:04.858083704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:53:04.867862408Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
ts=2026-05-12T22:53:05.451762781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41174"
level=warn ts=2026-05-12T22:53:09.187517948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:11.434011138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:53:11.438130709Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
ts=2026-05-12T22:53:11.494758644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:53:12.897791463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:53:12.905656957Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:53:14.072576805Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:53:14.074696748Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:53:14.187525961Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:53:14.187571264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:14.187604553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:53:14.187620574Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:53:14.187629279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:53:14.187735166Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-12T22:53:14.187745319Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:53:14.187764475Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.502µs success=true
level=debug ts=2026-05-12T22:53:14.187963296Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-05-12T22:53:15.120688636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49274"
level=info ts=2026-05-12T22:53:17.497148696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:53:17.505796937Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:53:17.52196825Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:53:18.49120433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:53:18.495328627Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:53:19.187680928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:19.426011873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:53:20.61968088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:53:20.623770444Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:53:20.855533761Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=info ts=2026-05-12T22:53:21.768245287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:53:21.771724448Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:53:22.616025527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:53:22.618454156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:53:22.619758995Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:53:22.621321198Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:53:22.916191745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:53:22.919704892Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:53:24.187569009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:24.188624339Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-12T22:53:25.076224637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45044"
level=info ts=2026-05-12T22:53:25.647682003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:53:25.651158524Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:53:25.975190366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:53:25.978605564Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:53:27.522260953Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-12T22:53:28.263559963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:53:28.270962957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:53:28.815350759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:53:28.818783036Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:53:29.187785749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:30.388223703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:53:30.391794856Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=debug ts=2026-05-12T22:53:30.855003357Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-12T22:53:33.794006132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:33.798243731Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:33.798263291Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:53:33.79829008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:33.801902212Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:53:34.18701251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:34.188257707Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-12T22:53:35.255557215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:53:35.257451488Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:53:35.277806782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:53:35.284456103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:53:35.634347553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:53:35.637946786Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:53:36.510388151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:53:36.513855059Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:53:37.306078752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:53:37.31312045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:53:37.521890286Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-12T22:53:39.187674928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:40.856956651Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:53:40.856985639Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=9
ts=2026-05-12T22:53:41.495850727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:53:42.00579967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:53:42.009442263Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=info ts=2026-05-12T22:53:43.638532824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:53:43.642756782Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:53:43.896824016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:53:43.901107054Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T22:53:44.187692216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:53:44.187700518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:44.187735559Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:53:44.187747611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:53:44.189678089Z caller=checkpoint.go:372 msg="writing series" size="872 kB"
level=debug ts=2026-05-12T22:53:47.521557674Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T22:53:47.96076948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:47.965760211Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:53:49.188104995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:49.875531089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:53:49.879054947Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:53:50.855181Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:53:50.855206656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:53:50.98570251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:53:50.989914651Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:53:50.989938146Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:53:50.989961549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:53:50.99327693Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:53:51.264119482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:53:51.267698232Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:53:51.316746892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:53:51.320171134Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T22:53:52.178532924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:53:54.187268905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:54.188425909Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
ts=2026-05-12T22:53:55.078982482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48046"
level=info ts=2026-05-12T22:53:56.411119936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:53:56.414723497Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:53:57.521850705Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-12T22:53:59.188039878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:59.99738773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:54:00.00342461Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:54:00.855059306Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:54:02.880114316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:54:02.888612353Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:54:03.703216416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:54:03.710161064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:54:04.187865804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:04.189276856Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=info ts=2026-05-12T22:54:05.447313932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:54:05.4513253Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-12T22:54:05.456394754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60834"
level=info ts=2026-05-12T22:54:06.653396634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:54:06.657749436Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:54:07.522206962Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-12T22:54:07.822970968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:54:07.827145395Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:54:07.827163675Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:07.827184329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:54:07.830999019Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:54:08.00272545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:54:08.006730584Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:54:09.188028342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:10.855430419Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-12T22:54:11.498798091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=info ts=2026-05-12T22:54:11.572524873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:54:11.576402165Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:54:14.072582918Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:54:14.074703777Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:54:14.187602043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:54:14.187625918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:14.187637215Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:54:14.187652114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:54:14.190568167Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-12T22:54:14.190583949Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=7
level=info ts=2026-05-12T22:54:14.313508688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:54:14.317647099Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
ts=2026-05-12T22:54:15.125102679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40504"
level=debug ts=2026-05-12T22:54:17.522289461Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-12T22:54:18.320500826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:54:18.326278132Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:54:18.599994026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:54:18.603586579Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:54:19.187506254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:54:19.908532162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:54:19.912221868Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:54:19.91224967Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:19.912280402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:54:19.915726935Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:54:20.87188842Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-12T22:54:20.871921492Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-05-12T22:54:21.186789073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:54:21.192191623Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:54:22.29748203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:54:22.301444262Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:54:22.470990607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:54:22.474370714Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:54:23.348936971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:54:23.356874357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:54:24.187055536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:24.188315544Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=info ts=2026-05-12T22:54:24.852801327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:54:24.859586102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:54:24.859609851Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:24.859639247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:54:24.866091267Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:54:25.08220902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:46494"
level=info ts=2026-05-12T22:54:25.275938427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:54:25.27935055Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:54:25.47120456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:54:25.475774817Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:54:26.940546406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:54:27.365321688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:54:27.522587762Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:54:27.52261021Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=info ts=2026-05-12T22:54:27.796720561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:54:27.800181731Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:54:29.18788024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:30.855417835Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:54:32.169514577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:54:32.170574083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:54:32.173061794Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:54:32.175138123Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:54:33.16702071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:54:33.170416874Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:54:33.566605645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:54:33.573021694Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:54:34.187530095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:34.188785776Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
ts=2026-05-12T22:54:35.457853259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52500"
level=debug ts=2026-05-12T22:54:37.522521932Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-12T22:54:38.022740584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:54:38.026379995Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:54:38.981217205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:54:38.985337609Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:54:39.187948655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:54:39.503934312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:54:39.511588815Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:54:39.649022944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:54:39.652845082Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=info ts=2026-05-12T22:54:39.699128778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:54:39.702587632Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:54:40.855537056Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
ts=2026-05-12T22:54:41.500888405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:54:44.187482674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:54:44.187508866Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:54:44.187519557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:54:44.187536349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:44.188645449Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:54:44.28626633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:54:44.289725894Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:54:44.289742959Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:44.289765303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:54:44.292619734Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:54:46.117403447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:54:46.124526562Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:54:47.521368077Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:54:48.053802137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:54:48.057345812Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:54:48.057368817Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:48.057393391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:54:48.060387857Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=warn ts=2026-05-12T22:54:49.187721846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:50.855562193Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-12T22:54:50.855586202Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-12T22:54:52.192032586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:54:52.197601603Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:54:52.319048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:54:52.323545786Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:54:53.995433691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:54:53.99890657Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:54:54.09798073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:54:54.105151082Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:54:54.18738717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:54.188460963Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-12T22:54:55.085650714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34892"
level=info ts=2026-05-12T22:54:55.188398086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:54:55.192392091Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=info ts=2026-05-12T22:54:57.42496258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:54:57.432906267Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:54:57.52225119Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-12T22:54:58.615175283Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:54:58.793691248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:54:58.797201353Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T22:54:59.188057005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:54:59.799661661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:54:59.803147947Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:55:00.855028299Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-12T22:55:02.893983369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:02.898749755Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:55:04.187575369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:04.191707899Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-12T22:55:04.191730552Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=4
level=info ts=2026-05-12T22:55:05.115118121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:55:05.118572059Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:55:05.697348443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:55:05.701371538Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:55:07.521534726Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=warn ts=2026-05-12T22:55:09.187499946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:10.855281008Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-12T22:55:11.502665336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:55:11.777104973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:55:11.784562589Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:55:14.072818335Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:55:14.073879104Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:55:14.186990976Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:55:14.187000598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:14.187019888Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:55:14.187028078Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:55:14.188154024Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-12T22:55:15.297056302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:55:15.306832447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:55:15.750936699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:55:15.754561918Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:55:16.283387579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:55:16.287503038Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:55:16.491137786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:55:16.494805492Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:55:16.667335073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:55:16.670590824Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:55:17.521650578Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-12T22:55:17.856380389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:55:17.860462825Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:55:19.187032287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:19.605125776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:55:19.608635912Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:55:20.160842229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:55:20.165813125Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:55:20.855610336Z caller=checkpoint.go:372 msg="writing series" size="518 kB"
level=info ts=2026-05-12T22:55:21.100758697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:55:21.10468752Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:55:21.542820987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:55:21.54649626Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:55:22.310879345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:55:22.316429659Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=info ts=2026-05-12T22:55:23.088650151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:55:23.097112725Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:55:23.097134959Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:23.097158361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:55:23.10421182Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:55:23.139283789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:55:23.142501251Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:55:24.165332081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:55:24.169039986Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=warn ts=2026-05-12T22:55:24.187182693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:24.188249539Z caller=checkpoint.go:372 msg="writing series" size="993 B"
level=info ts=2026-05-12T22:55:24.583195899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:55:24.58712973Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:55:27.522123112Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-12T22:55:27.789229825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:55:28.839321153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:55:29.187746486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:30.855568485Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-12T22:55:30.855593509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-12T22:55:34.187947286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:34.188009768Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-12T22:55:35.125028501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:55:35.132736053Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:55:36.732433484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:55:36.73922373Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:55:36.739247743Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:36.73927115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:55:36.745337361Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:55:37.522442308Z caller=checkpoint.go:372 msg="writing series" size="558 kB"
level=info ts=2026-05-12T22:55:38.832497642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:55:38.836409302Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:55:39.037988901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:55:39.041444416Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:55:39.187917629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:39.925752523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:55:39.929092893Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=debug ts=2026-05-12T22:55:40.856018053Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=debug ts=2026-05-12T22:55:40.856040813Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:55:41.503988615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:55:44.187397268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:55:44.187399049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:44.187426741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:55:44.187435053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:55:44.188686847Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=info ts=2026-05-12T22:55:44.282320298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:55:44.290484075Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:55:45.131402931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59164"
level=info ts=2026-05-12T22:55:45.251184072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:55:45.254675384Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:55:46.992556536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:55:46.996086149Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:55:47.521704603Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=info ts=2026-05-12T22:55:49.150437218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:55:49.153918953Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:55:49.187195834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:50.149121234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:55:50.156098791Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:55:50.354692205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:55:50.364343887Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:55:50.855591134Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:55:53.384941857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:55:53.394489545Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:55:54.187728384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:54.188944416Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=debug ts=2026-05-12T22:55:54.188959197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-12T22:55:57.525519341Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-12T22:55:57.525563485Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=debug ts=2026-05-12T22:55:58.940262349Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:55:59.187972453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:59.398212685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:59.403060118Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:59.403096154Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:59.403120393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:59.406747334Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:55:59.941366943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:55:59.944703323Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:56:00.681019226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:56:00.684446788Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:56:00.854716535Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:56:00.937134782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:56:00.944026617Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:56:01.338359921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:56:01.342524337Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:56:01.588341787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:56:01.592148187Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:56:03.419460555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:56:03.423420596Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:56:04.187638628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:04.188839076Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-12T22:56:04.267308532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:56:04.270580323Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=debug ts=2026-05-12T22:56:07.521759149Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=info ts=2026-05-12T22:56:07.872477919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:56:07.876014443Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:56:08.406537173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:56:08.410004239Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:56:08.606031036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:56:08.609381602Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:56:08.700741064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:56:08.704022312Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:56:08.707091451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:56:08.710711637Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:56:08.710729151Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:08.710755282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:56:08.714244456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T22:56:09.187317318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:56:09.202440534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:56:09.208828665Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:56:10.855060591Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T22:56:10.855101872Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:56:11.475390264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:56:11.479554602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-12T22:56:11.505869734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:56:11.979848588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:56:11.98340223Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:56:14.072359504Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:56:14.074461476Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:56:14.187269554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:56:14.187293921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:56:14.187317352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:56:14.187325817Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:56:14.188648361Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=info ts=2026-05-12T22:56:16.622573345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:56:16.626088909Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-12T22:56:17.521727836Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:56:17.793588951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:56:17.799357159Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:56:18.510911932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:56:18.514657436Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:56:19.187764377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:20.855962768Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:56:20.85598369Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-12T22:56:21.152299478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:56:21.15590276Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=warn ts=2026-05-12T22:56:24.187422092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:24.241357784Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-12T22:56:24.241389634Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
ts=2026-05-12T22:56:25.095218806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:43524"
level=info ts=2026-05-12T22:56:25.249086363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:56:25.255395424Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:56:27.522208396Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-12T22:56:27.568482163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:56:27.568534638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=56c5a98ccd161bbd immediate=false num_chunks=1 total_comp="362 kB" avg_comp="362 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" idle=1 labels="{stream=\"build-e2e-jhea\"}"
level=error ts=2026-05-12T22:56:27.578386688Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/56c5a98ccd161bbd/19e1e1b0190%3A19e1e30cbb0%3Ac1ec22d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-jhea\"}"
level=debug ts=2026-05-12T22:56:28.982946034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:56:29.187041272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:30.85525692Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-05-12T22:56:32.568999322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:56:32.573022459Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:56:34.187554715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:34.190785597Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:56:34.190803214Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-05-12T22:56:34.665520002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:56:34.669423636Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:56:34.669451143Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:34.669482524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:56:34.672575085Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:56:36.127838969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:56:36.132644648Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:56:36.347123286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:56:36.350711766Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:56:37.15102547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:56:37.155170462Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:56:37.522218792Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:56:38.041632225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:56:38.049558995Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:56:38.761627171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=56c5a98ccd161bbd immediate=false num_chunks=1 total_comp="385 kB" avg_comp="385 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-jhea\"}"
level=error ts=2026-05-12T22:56:38.765352617Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/56c5a98ccd161bbd/19e1e1b0190%3A19e1e30cbb0%3Ac1ec22d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-jhea\"}"
level=warn ts=2026-05-12T22:56:39.187342063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:56:39.293123248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:56:39.30217385Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:56:40.855947946Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
ts=2026-05-12T22:56:41.507438524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:56:44.187871635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:56:44.187875782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:44.187912154Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:56:44.18792018Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:56:44.187990821Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:56:44.662355646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:56:44.666236389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T22:56:45.135864214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48692"
level=info ts=2026-05-12T22:56:47.45274211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:56:47.45749513Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:56:47.52183274Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-12T22:56:48.64795038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:56:48.651756185Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=warn ts=2026-05-12T22:56:49.187667007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:56:49.30164885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:56:49.305058558Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:56:49.410813914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:56:49.414883821Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:56:49.414904433Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:49.414925515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:56:49.417947134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:56:50.254818505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:56:50.263967527Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:56:50.857693189Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-12T22:56:50.857715953Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=5
level=info ts=2026-05-12T22:56:50.98409974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:56:50.987591361Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:56:50.987614208Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:50.987636298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:56:50.990814898Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:56:51.39919386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:56:51.402987663Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:56:51.403008584Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:51.403030132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:56:51.406275783Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=debug ts=2026-05-12T22:56:51.662888222Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:56:52.086344211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:56:52.090795517Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:56:54.18704103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:54.188127383Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:56:56.610209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:56:56.615851195Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:56:56.814355736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:56:56.817730353Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:56:57.072645183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:56:57.080051298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:56:57.521811652Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-12T22:56:57.649988478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:56:57.653608031Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:56:57.653632801Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:57.653656729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:56:57.656585433Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:56:58.379841576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:56:58.38321272Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:56:58.383233627Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:58.383257003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:56:58.38611402Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:56:59.040692704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:56:59.044057145Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=warn ts=2026-05-12T22:56:59.187132087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:00.206546676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:57:00.213878311Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:57:00.856557734Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:57:00.856581756Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-12T22:57:01.042120906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:57:01.045546904Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:57:01.629310553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:57:01.6331377Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=info ts=2026-05-12T22:57:01.723464886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:57:01.72752568Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-12T22:57:04.187268883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:04.188395653Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T22:57:04.322842542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:57:04.326478005Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:57:05.678611037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:57:05.682293269Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=info ts=2026-05-12T22:57:05.85354445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:57:05.860042297Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:57:07.522031727Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-12T22:57:09.187814987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:09.3064238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:57:09.310170372Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:57:09.377134352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:57:09.380675908Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:57:10.153459489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=56c5a98ccd161bbd immediate=false num_chunks=1 total_comp="385 kB" avg_comp="385 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-jhea\"}"
level=error ts=2026-05-12T22:57:10.157733518Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/56c5a98ccd161bbd/19e1e1b0190%3A19e1e30cbb0%3Ac1ec22d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-jhea\"}"
level=debug ts=2026-05-12T22:57:10.855798044Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-12T22:57:10.855819748Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-12T22:57:11.510181037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:57:12.044733006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:57:12.62768955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:57:12.631548982Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:57:13.256413566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:57:13.259884387Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:57:14.073031888Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:57:14.07408966Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:57:14.187123923Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:57:14.187156211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:57:14.187171672Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:57:14.18719387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:14.188631599Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
ts=2026-05-12T22:57:15.137205098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51024"
level=info ts=2026-05-12T22:57:16.782280387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:57:16.785803873Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:57:17.221995811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:57:17.525201613Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-12T22:57:19.187341231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:20.855282121Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-12T22:57:20.923465422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:57:20.93149947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:57:23.524209918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:57:23.527863151Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:57:23.52788443Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:57:23.527908519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:57:23.530667045Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:57:24.18723581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:24.188381143Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-12T22:57:25.228583724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:57:25.232316458Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:57:25.232334786Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:57:25.232354871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:57:25.235354259Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:57:25.296662878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:57:25.30011166Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:57:25.955757199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:57:25.95983933Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:57:26.174642774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:57:26.179064462Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:57:27.097852751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:57:27.107741599Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:57:27.522559509Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=debug ts=2026-05-12T22:57:27.522581115Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-12T22:57:29.187805583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:30.855140092Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:57:32.645361229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:57:32.64984018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:57:34.187159819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:34.18836398Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-12T22:57:35.765735936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:57:35.770240658Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:57:37.213023172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:57:37.216667971Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:57:37.52171932Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-12T22:57:38.734121256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:57:38.738954114Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T22:57:39.056749877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:57:39.066033861Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:57:39.186493053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:57:39.187566072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:57:39.189994826Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:57:39.832875176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:57:39.836593962Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:57:40.712976499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:57:40.717251154Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:57:40.855606813Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:57:41.392284319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:57:41.395829595Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-12T22:57:41.5125148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:57:41.736870815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:57:41.740463983Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:57:43.870117314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:57:43.874150705Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:57:44.053508703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:57:44.05747075Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:57:44.187480735Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:57:44.187490885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:44.187516483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:57:44.187525049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:57:44.188563081Z caller=checkpoint.go:384 msg="flushing series" totalSize="212 kB" series=4
level=info ts=2026-05-12T22:57:44.194048014Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-12T22:57:44.195294424Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007377561s
level=info ts=2026-05-12T22:57:44.599525237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:57:44.604535464Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:57:46.080836436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:57:46.084311234Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:57:46.084329298Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:57:46.084351025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:57:46.087242223Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=info ts=2026-05-12T22:57:46.606590716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:57:46.610258334Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T22:57:46.783140548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:57:47.708720898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:57:47.712305086Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=warn ts=2026-05-12T22:57:49.187537727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:49.493050697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:57:49.496741654Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:57:49.496759396Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:57:49.496779427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:57:49.499701887Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=info ts=2026-05-12T22:57:53.715203012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:57:53.724358296Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:57:53.724378723Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:57:53.724421937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:57:53.731673534Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:57:53.89134322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:57:53.898505629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:57:54.187847763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:57:55.105038435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45750"
level=info ts=2026-05-12T22:57:57.456193136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:57:57.461681823Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=info ts=2026-05-12T22:57:57.749653982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:57:57.755977583Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:57:59.187118972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:01.777871579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:58:01.781654621Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:58:01.915093303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=56c5a98ccd161bbd immediate=false num_chunks=1 total_comp="385 kB" avg_comp="385 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-jhea\"}"
level=error ts=2026-05-12T22:58:01.919239495Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/56c5a98ccd161bbd/19e1e1b0190%3A19e1e30cbb0%3Ac1ec22d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-jhea\"}"
level=info ts=2026-05-12T22:58:02.681704814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:58:02.685750128Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=info ts=2026-05-12T22:58:03.857973377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:58:03.861660578Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-12T22:58:04.187652737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:05.472608568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:58:05.476449814Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:58:05.703053976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:58:05.710297164Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:58:07.463175437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:58:07.467060684Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:58:07.46710021Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:58:07.46712218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:58:07.471010861Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:58:08.052543307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:58:08.054462929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:58:09.187285356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:10.912976565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:58:10.920602096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:58:11.398093356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:58:11.406093943Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:58:11.513652339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=debug ts=2026-05-12T22:58:14.072686612Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:58:14.074798277Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:58:14.187658778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:58:14.187682826Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:58:14.187690661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:58:14.187684016Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:58:14.187740559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:14.187904895Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-12T22:58:14.187914252Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:58:14.187933693Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.463µs success=true
level=debug ts=2026-05-12T22:58:14.188129657Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-12T22:58:15.05932843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:58:15.062937405Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:58:15.85153965Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:58:16.524538921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:58:16.528350929Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=info ts=2026-05-12T22:58:17.162208443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:58:17.166230832Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-12T22:58:17.522376216Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:58:19.188046225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:20.58929027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:58:20.594427113Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:58:20.855283004Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=info ts=2026-05-12T22:58:22.660368475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:58:22.664205702Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:58:24.187690384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:24.188720676Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-12T22:58:24.420008912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:58:24.423373666Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:58:24.920818096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:58:24.925013523Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:58:25.069942452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:58:25.07418641Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:58:25.10234624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:58:25.10600943Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=info ts=2026-05-12T22:58:25.744444329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:58:25.748408773Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=info ts=2026-05-12T22:58:26.252822716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:58:26.257522673Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:58:26.257545497Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:58:26.257572523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:58:26.261407704Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:58:27.522133054Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-12T22:58:29.18772307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:30.789014193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:58:30.792790641Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T22:58:30.854943475Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-12T22:58:31.640294258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:58:31.644153231Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:58:32.807688149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:58:32.811014429Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:58:34.187642677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:34.188784773Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
ts=2026-05-12T22:58:35.472969488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42266"
level=info ts=2026-05-12T22:58:35.672121956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:58:35.675476312Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=info ts=2026-05-12T22:58:35.883992058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:58:35.892493768Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=debug ts=2026-05-12T22:58:37.522385357Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-12T22:58:38.64539908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:58:38.649985355Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T22:58:39.1876983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:40.515195581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-12T22:58:40.518916452Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e1e12f60a%3A19e1e1325cc%3A90f55db1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-12T22:58:40.536975818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41c387cb53eaaa25 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-hvns\"}"
level=error ts=2026-05-12T22:58:40.540670224Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41c387cb53eaaa25/19e1e1ad0fd%3A19e1e2c158b%3Afbe41850?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hvns\"}"
level=debug ts=2026-05-12T22:58:40.857166564Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-12T22:58:40.857190324Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=9
ts=2026-05-12T22:58:41.514853283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-84031014 10.129.0.73:7946"
level=info ts=2026-05-12T22:58:41.896119802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:58:41.899586124Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:58:41.89960526Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:58:41.899628042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:58:41.902744221Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:58:42.54274076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:58:42.548794857Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:58:44.08809635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-12T22:58:44.091453372Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1e1ac3f3%3A19e1e1c5e1b%3A2a09a125?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-12T22:58:44.186938919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:58:44.186971352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:58:44.186982329Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:58:44.187019957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:44.189591668Z caller=checkpoint.go:372 msg="writing series" size="874 kB"
level=info ts=2026-05-12T22:58:44.823289434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:58:44.830484521Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:58:46.077458065Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:58:47.521824975Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T22:58:48.086657277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:58:48.090890581Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:58:49.187447809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:50.855640605Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-12T22:58:50.855662665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-12T22:58:51.159571218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:58:51.163134487Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e12f60a%3A19e1e130f1b%3A9f3f7b82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:58:53.010282582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:58:53.024778393Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:58:53.747881686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:58:53.754571473Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:58:54.187493039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:54.188679702Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
ts=2026-05-12T22:58:55.111131668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41700"
level=info ts=2026-05-12T22:58:55.194905584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:58:55.201816994Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e129453%3A19e1e133908%3A3bf29afe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:58:57.522297316Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:58:58.803891025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:58:58.807710665Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e129664%3A19e1e1325cc%3Aa21b4b77?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:58:59.187352811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:00.855144469Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:59:01.324825722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=56c5a98ccd161bbd immediate=false num_chunks=1 total_comp="385 kB" avg_comp="385 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-jhea\"}"
level=error ts=2026-05-12T22:59:01.331738843Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/56c5a98ccd161bbd/19e1e1b0190%3A19e1e30cbb0%3Ac1ec22d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-jhea\"}"
level=info ts=2026-05-12T22:59:02.418058973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="389 kB" avg_comp="389 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-12T22:59:02.422032504Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1e1293f9%3A19e1e42d8c9%3A9b04a101?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-12T22:59:03.075955919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="471 kB" avg_comp="471 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T22:59:03.080174308Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e12ad62%3A19e1e362e98%3A79d4e1a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T22:59:03.798472091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:59:04.187315429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:04.188709697Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=info ts=2026-05-12T22:59:05.415976236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:05.424327921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:05.424350302Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:59:05.424374571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:05.431149248Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:59:05.474252626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46718"
level=debug ts=2026-05-12T22:59:07.521661393Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T22:59:09.187502459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:59:10.278490042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="584 kB" avg_comp="584 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T22:59:10.282915186Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e129424%3A19e1e361bf3%3A45a4c5e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T22:59:10.573819679Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:59:10.772362045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba1c11b178a9ace8 immediate=false num_chunks=1 total_comp="205 kB" avg_comp="205 kB" total_uncomp="906 kB" avg_uncomp="906 kB" full=1 labels="{stream=\"stat-rep-czod\"}"
level=error ts=2026-05-12T22:59:10.776353993Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba1c11b178a9ace8/19e1e1affdd%3A19e1e24c674%3A9aa605c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-czod\"}"
level=debug ts=2026-05-12T22:59:10.855500354Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-12T22:59:11.515917781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=info ts=2026-05-12T22:59:11.655788924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:59:11.659355242Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e12f60a%3A19e1e134dcc%3Aa2b720dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:59:14.07275265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.63:11211 count=1
level=info ts=2026-05-12T22:59:14.07485224Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:59:14.18571158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:59:14.187053157Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:59:14.1870965Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:59:14.187110986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:59:14.187116136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:14.189145824Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-12T22:59:14.189160731Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=7
level=error ts=2026-05-12T22:59:14.189273884Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e12acec%3A19e1e12f60b%3A207c86db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:59:15.025679261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:59:15.029907151Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e129670%3A19e1e40b81d%3A524975f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-12T22:59:15.147862207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42728"
level=info ts=2026-05-12T22:59:15.20943947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab2416cbc5609b8 immediate=false num_chunks=1 total_comp="53 kB" avg_comp="53 kB" total_uncomp="185 kB" avg_uncomp="185 kB" full=1 labels="{stream=\"build-e2e-dqrh\"}"
level=error ts=2026-05-12T22:59:15.21299714Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab2416cbc5609b8/19e1e1be039%3A19e1e2c6a99%3A49cb0721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dqrh\"}"
level=debug ts=2026-05-12T22:59:17.522425443Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-12T22:59:18.293839791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:59:18.29812682Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e12acaa%3A19e1e12f60b%3Af5b4f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:59:19.187453925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:59:19.852199644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="581 B" avg_comp="581 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:59:19.856101475Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e12f60a%3A19e1e12f60b%3A1d36c8ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:59:20.421606272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc4b9541f8cdb3 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-hplv\"}"
level=error ts=2026-05-12T22:59:20.425490002Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc4b9541f8cdb3/19e1e1ae637%3A19e1e1c97d1%3A6294fd5c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-hplv\"}"
level=debug ts=2026-05-12T22:59:20.87323556Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-12T22:59:20.87325892Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=2
level=info ts=2026-05-12T22:59:21.16682285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:59:21.170944986Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e12f60a%3A19e1e28f4c4%3A86f8db12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T22:59:22.577897265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:22.586659451Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e129bd4%3A19e1e2d600b%3Ab7cad13b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:59:24.187790223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:24.189036699Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-12T22:59:24.381274013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="154 kB" avg_uncomp="154 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:59:24.384785799Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e130f19%3A19e1e14f2c9%3Abdb859ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:59:25.267117855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:59:25.276509121Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=error ts=2026-05-12T22:59:25.276532767Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:59:25.276557516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=38fe59c06035f0e3 immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-oref\"}"
level=error ts=2026-05-12T22:59:25.284874962Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38fe59c06035f0e3/19e1e1aeeeb%3A19e1e2776da%3A5dacef43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-oref\"}"
level=info ts=2026-05-12T22:59:26.548449757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="441 kB" avg_comp="441 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:59:26.552683221Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e12943b%3A19e1e52926d%3A7e5e7f47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:59:26.967873421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=629b879b9e3520e9 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-ecqf-managed\"}"
level=error ts=2026-05-12T22:59:26.971719695Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/629b879b9e3520e9/19e1e25ff6c%3A19e1e297994%3A979eaeee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-ecqf-managed\"}"
level=debug ts=2026-05-12T22:59:27.524613901Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:59:27.524642386Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=info ts=2026-05-12T22:59:28.176556105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:59:28.184321463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e1293fd%3A19e1e1acd81%3Aa209bd1e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:59:29.187572198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:59:30.583707497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:59:30.591363213Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e12975d%3A19e1e5a0fbb%3A220d3fce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:59:30.855024341Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:59:33.023928733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:59:33.034166479Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e130f19%3A19e1e1325cc%3Aa0f8773f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:59:34.187155654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:34.18838936Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=debug ts=2026-05-12T22:59:35.08555252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:59:35.342329538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:59:35.349775877Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e12f5e4%3A19e1e4d97ab%3Aaef9b94e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-12T22:59:35.475273024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56560"