level=info ts=2026-05-13T16:53:40.584481198Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T16:53:40.584520524Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-13T16:53:40.584946761Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-13T16:53:40.586017246Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T16:53:40.58644079Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T16:53:40.586537138Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.101 inf=eth0
level=info ts=2026-05-13T16:53:40.58715427Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-dbcdb230
ts=2026-05-13T16:53:40.587229814Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T16:53:40.587244931Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.101 advertisePort=7946
level=debug ts=2026-05-13T16:53:40.587269862Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.101 advertisePort=7946
level=info ts=2026-05-13T16:53:40.587421536Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-13T16:53:40.593421183Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T16:53:40.593450522Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-05-13T16:53:40.59705121Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T16:53:40.597118962Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T16:53:40.597148958Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=9.729355ms
level=error ts=2026-05-13T16:53:40.597161419Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T16:53:40.597172462Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-13T16:53:40.598004121Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-13T16:53:40.59820521Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:53:40.59847423Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-13T16:53:40.598490892Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-13T16:53:40.598489028Z
level=debug ts=2026-05-13T16:53:40.598502839Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-13T16:53:40.598529373Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-13T16:53:40.598547383Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-13T16:53:40.598553588Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-13T16:53:40.598552554Z
level=debug ts=2026-05-13T16:53:40.598570009Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-05-13T16:53:40.60040151Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T16:53:40.600418235Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T16:53:40.600469759Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T16:53:40.600574969Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.101 inf=eth0
level=debug ts=2026-05-13T16:53:40.61977087Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=info ts=2026-05-13T16:53:40.619864613Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-13T16:53:40.619903276Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-13T16:53:40.620002922Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-13T16:53:40.62005514Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-13T16:53:40.619981513Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-13T16:53:40.620350396Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-05-13T16:53:40.620691816Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-13T16:53:40.620727916Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-13T16:53:40.620736796Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-13T16:53:40.620944545Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-13T16:53:40.620958166Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-13T16:53:40.620952291Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-13T16:53:40.620977436Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-13T16:53:40.621048142Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-13T16:53:40.62109821Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-13T16:53:40.621103945Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-05-13T16:53:40.621108791Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-13T16:53:40.621113933Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-13T16:53:40.621121296Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-13T16:53:40.621132009Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-13T16:53:40.621137518Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-05-13T16:53:40.621142502Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-13T16:53:40.62122813Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=info ts=2026-05-13T16:53:40.621240725Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-13T16:53:40.621266472Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-13T16:53:40.621275093Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T16:53:40.621332125Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=33.186µs success=true
level=info ts=2026-05-13T16:53:40.621348175Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-13T16:53:40.621443576Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=179.216µs errors=false
level=info ts=2026-05-13T16:53:40.621458899Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-13T16:53:40.621606646Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=337.109µs errors=false
level=info ts=2026-05-13T16:53:40.621625877Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-13T16:53:40.621633289Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=368.942µs
level=info ts=2026-05-13T16:53:40.621685128Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-13T16:53:40.621694684Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T16:53:40.621721671Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-13T16:53:40.621727105Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=12.500965795s
level=debug ts=2026-05-13T16:53:40.621830073Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-13T16:53:40.621842174Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-05-13T16:53:40.621840658Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-13T16:53:40.621868471Z caller=loki.go:615 msg="Loki started" startup_time=119.16448ms
level=debug ts=2026-05-13T16:53:40.621938889Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-13T16:53:41.742116484Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T16:53:41.742143957Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T16:53:44.132592781Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T16:53:44.132616951Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T16:53:45.622063772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:53:48.68192421Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T16:53:48.681946Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T16:53:50.622004486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:53:55.622418782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:53:57.341466665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.54:7946"
ts=2026-05-13T16:53:57.34323689Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.53:7946: dial tcp 10.130.0.53:7946: connect: connection refused"
level=info ts=2026-05-13T16:53:57.343253015Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=16.746074617s
level=warn ts=2026-05-13T16:54:00.622817595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:54:04.594452732Z caller=logging.go:137 msg="GET /ready (503) 95.845µs"
level=warn ts=2026-05-13T16:54:05.62243634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:54:10.622818099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T16:54:10.622870963Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=warn ts=2026-05-13T16:54:10.622861235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:54:10.622895415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T16:54:14.593535495Z caller=logging.go:137 msg="GET /ready (503) 78.909µs"
level=warn ts=2026-05-13T16:54:15.622286731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:54:17.039622891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:51064"
ts=2026-05-13T16:54:20.340623126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T16:54:20.622644472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:54:25.622316003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:54:26.523923946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T16:54:26.523988984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:54:26.530384383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T16:54:30.622610156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:54:35.622941763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:54:39.592534751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:54:39.598718348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T16:54:40.595550748Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T16:54:40.59866099Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:54:40.621948595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T16:54:40.621960074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:54:40.621981472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:54:40.621993114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T16:54:41.04373975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T16:54:44.787722152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:60550"
level=warn ts=2026-05-13T16:54:45.6225769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:54:47.041047027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:35746"
ts=2026-05-13T16:54:50.343081619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T16:54:50.622096958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:54:52.317380358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:54:55.622757631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:55:00.569222455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:55:00.579491822Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T16:55:00.622477682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:55:01.246742073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T16:55:05.622701876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:55:10.621836177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:55:10.621897983Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:55:10.621910076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:55:10.62194959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:55:15.622329706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:55:17.042534508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:47758"
ts=2026-05-13T16:55:20.344485124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T16:55:20.622334443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:55:24.07451034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T16:55:24.814069505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:55:25.622637799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:55:30.621951078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:55:35.62221092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:55:40.594793597Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T16:55:40.598896335Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:55:40.622291879Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:55:40.622325721Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T16:55:40.622309767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:55:40.622333201Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T16:55:44.791185954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:38656"
level=warn ts=2026-05-13T16:55:45.622660139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:55:50.346635237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T16:55:50.622265902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:55:52.895287957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T16:55:53.248046346Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:55:55.621991908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:55:59.173994419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:55:59.181474795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T16:56:00.622631005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:56:05.622324249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:56:10.621857813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:56:10.621940117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:56:10.621948335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:56:10.621967051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:56:15.622463766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:56:17.046460128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:57402"
ts=2026-05-13T16:56:20.3485277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T16:56:20.622728533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:56:24.982137972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T16:56:24.984374462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T16:56:25.622231702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:56:28.619366766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:56:30.621918044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:56:35.622825915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:56:40.596249698Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T16:56:40.598376934Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:56:40.62275059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:56:40.62277614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T16:56:40.622784717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:56:40.622787606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:56:45.622408545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:56:46.26660436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:56:46.2736428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T16:56:47.04743103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:33404"
ts=2026-05-13T16:56:48.201008296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34590"
ts=2026-05-13T16:56:50.350487658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T16:56:50.622422117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:56:54.639523678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:56:55.622032122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:57:00.622102091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:57:05.622749942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:57:10.384509971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T16:57:10.622392659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T16:57:10.622408096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:57:10.622430674Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:57:10.622439848Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:57:15.622212338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:57:18.203156046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35068"
ts=2026-05-13T16:57:20.351387039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T16:57:20.622777582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:57:25.622722225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:57:30.094616889Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:57:30.622249248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:57:31.356714181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:57:31.365922663Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T16:57:35.622190012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:57:40.595424387Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T16:57:40.598531593Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:57:40.621810839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:57:40.621827176Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:57:40.621835073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:57:40.621906228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:57:45.622622678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:57:47.050676145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:44726"
ts=2026-05-13T16:57:50.352688362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T16:57:50.622720152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:57:55.163666367Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:57:55.622173707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:58:00.62251508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:58:05.622229434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:58:10.622435039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:58:10.622462113Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:58:10.62246986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:58:10.622482962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T16:58:15.622485623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T16:58:18.20639179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53780"
ts=2026-05-13T16:58:20.354210406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T16:58:20.622526807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:58:20.713243901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T16:58:25.622214922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:58:26.874213578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:58:30.622341511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:58:30.822203162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:58:30.829522223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T16:58:35.62249829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:58:40.595605769Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T16:58:40.598736235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:58:40.622056913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:58:40.62207536Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T16:58:40.622085251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:58:40.622102382Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T16:58:40.622186884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T16:58:40.622245262Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-13T16:58:40.622252843Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T16:58:40.622267305Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.89µs success=true
level=debug ts=2026-05-13T16:58:40.622471907Z caller=checkpoint.go:372 msg="writing series" size="8.1 kB"
level=debug ts=2026-05-13T16:58:44.977654729Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-05-13T16:58:45.622545177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:58:49.33293793Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
ts=2026-05-13T16:58:50.355119699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T16:58:50.622470673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:58:53.687396421Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-13T16:58:55.622350354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:58:58.042893965Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-13T16:58:58.416200643Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:59:00.622397702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:02.396945488Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-13T16:59:05.621905636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:06.75201794Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-05-13T16:59:10.109511507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T16:59:10.622477062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:59:10.62250295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:59:10.622510248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:59:10.622534815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:11.106773144Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=debug ts=2026-05-13T16:59:15.462369328Z caller=checkpoint.go:372 msg="writing series" size="413 kB"
level=warn ts=2026-05-13T16:59:15.62257421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T16:59:16.977919454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T16:59:16.98431734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T16:59:17.055843794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:53412"
ts=2026-05-13T16:59:18.209570747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59084"
level=debug ts=2026-05-13T16:59:19.81660389Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-13T16:59:20.356280157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T16:59:20.622638041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:24.171695133Z caller=checkpoint.go:372 msg="writing series" size="370 kB"
level=debug ts=2026-05-13T16:59:24.171717201Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=debug ts=2026-05-13T16:59:25.418967249Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T16:59:25.622127772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:28.526576863Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T16:59:30.622766583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:32.880958559Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-13T16:59:35.622816951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:37.235492573Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-13T16:59:40.596471418Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T16:59:40.598574477Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T16:59:40.621853939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T16:59:40.621887463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T16:59:40.621901208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T16:59:40.621945065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:41.590732489Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-13T16:59:45.621929288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:45.945852636Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=debug ts=2026-05-13T16:59:50.300453812Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T16:59:50.357750186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T16:59:50.622250617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:54.654926258Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T16:59:55.622102365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T16:59:56.564974578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T16:59:59.012310691Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T16:59:59.012382461Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=warn ts=2026-05-13T17:00:00.622721342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:03.364827931Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=debug ts=2026-05-13T17:00:05.614843559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T17:00:05.622897517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:07.720111344Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T17:00:10.62262683Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:00:10.622656517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:00:10.622666532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:00:10.622690589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:12.074483754Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=info ts=2026-05-13T17:00:13.68707994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:00:13.694329815Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:00:15.622532619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:16.42953354Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
ts=2026-05-13T17:00:18.213237886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51058"
ts=2026-05-13T17:00:20.359780807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:00:20.622406184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:20.784476871Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T17:00:24.681472567Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:00:25.139661602Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-13T17:00:25.622387765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:29.493972234Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=warn ts=2026-05-13T17:00:30.621968829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:33.849024347Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T17:00:35.622461421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:38.2033819Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-13T17:00:40.596725872Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:00:40.598865855Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T17:00:40.599119511Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T17:00:40.599163244Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976323 n=1
level=debug ts=2026-05-13T17:00:40.599210542Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976323 n=1
level=info ts=2026-05-13T17:00:40.622261153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:00:40.62226816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:40.622304474Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:00:40.622313397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:00:42.558177201Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T17:00:45.622434783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:46.913284115Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-13T17:00:47.059755405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:41360"
ts=2026-05-13T17:00:48.215587106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49972"
level=debug ts=2026-05-13T17:00:49.919702749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:00:50.361087279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:00:50.622600721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:51.267893744Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-13T17:00:55.622413161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:00:55.622508775Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-13T17:00:56.410807492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:00:56.420837356Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:00:56.420869992Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:00:56.420920712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:00:56.428732781Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:00:59.977870859Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-13T17:00:59.977924721Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-13T17:01:00.622423732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:04.332973389Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=warn ts=2026-05-13T17:01:05.622171132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:08.687598709Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T17:01:10.622582266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:01:10.62259412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:10.622612146Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:01:10.622623257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:01:13.0426056Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-13T17:01:14.809087338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:37288"
level=info ts=2026-05-13T17:01:15.358939876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:01:15.366786277Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:01:15.622485076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:01:17.060627614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:43030"
level=debug ts=2026-05-13T17:01:17.397458431Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-13T17:01:17.855635266Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:01:18.217830919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47286"
ts=2026-05-13T17:01:20.362661117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:01:20.622308023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:21.751955993Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-13T17:01:23.439486187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T17:01:23.441009078Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T17:01:25.622171496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:26.107163811Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=debug ts=2026-05-13T17:01:30.461261724Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-13T17:01:30.621994421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:34.816531532Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:01:35.622306509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:39.171850078Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:01:40.596127506Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:01:40.599235017Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:01:40.622586195Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:01:40.622604597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:01:40.62261691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:01:40.622631305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:42.090454316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:01:43.5263024Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
ts=2026-05-13T17:01:44.810836947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:40368"
level=warn ts=2026-05-13T17:01:45.622888873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:47.88195948Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-13T17:01:50.364172989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:01:50.622393446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:52.236056762Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-13T17:01:54.613519646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:01:54.622740057Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:01:55.622232862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:01:56.590551581Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T17:01:56.590575321Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-05-13T17:02:00.622627051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:00.94527371Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-13T17:02:05.300866327Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-13T17:02:05.62242001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:09.654761207Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T17:02:10.62183716Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:02:10.621863169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:02:10.621871221Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:02:10.621971249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:10.719690135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:02:14.01014064Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-13T17:02:15.62262009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:17.668948427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:02:18.221112405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53398"
level=debug ts=2026-05-13T17:02:18.365109008Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-13T17:02:20.365390912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:02:20.621949431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:22.719496906Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-13T17:02:25.622173455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:27.074571566Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T17:02:30.622290743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:31.429286085Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T17:02:35.622198505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:35.784323168Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-13T17:02:40.139236899Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-13T17:02:40.596523911Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:02:40.59864635Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T17:02:40.621914841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:02:40.621948343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:02:40.621971299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:02:40.621980856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:02:41.98658142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:02:41.99325979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:02:44.494059816Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
ts=2026-05-13T17:02:44.813428486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:44698"
level=debug ts=2026-05-13T17:02:45.47344199Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:02:45.622258084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:02:47.064889268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:60544"
level=debug ts=2026-05-13T17:02:48.440055088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:02:48.848392327Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
ts=2026-05-13T17:02:50.36660511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:02:50.622817862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:53.203152978Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-13T17:02:55.622229648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:02:57.558327003Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-13T17:03:00.622706077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:03:01.913653383Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-13T17:03:05.62195441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:03:06.271619428Z caller=checkpoint.go:372 msg="writing series" size="5.8 MB"
level=debug ts=2026-05-13T17:03:06.271640594Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.9 MB" series=16
level=warn ts=2026-05-13T17:03:10.622713904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:03:10.622741347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:03:10.62276788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:03:10.622785081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:03:10.631516158Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-13T17:03:10.635525188Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013107341s
level=debug ts=2026-05-13T17:03:14.673618939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:03:14.814278633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:48052"
level=warn ts=2026-05-13T17:03:15.622716342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:03:20.368051278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:03:20.622166689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:03:25.622015855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:03:30.622379855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:03:33.494919675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:03:33.50233693Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:03:35.622068212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:03:40.594709263Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:03:40.598909214Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:03:40.622258245Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:03:40.622273713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:03:40.622268922Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:03:40.622289432Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:03:40.622316042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:03:40.622389021Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-13T17:03:40.622395554Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:03:40.622413145Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.266µs success=true
level=debug ts=2026-05-13T17:03:40.622582075Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T17:03:42.124484301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:03:44.978024857Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-13T17:03:45.622537805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:03:46.367455122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:03:49.333079727Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
ts=2026-05-13T17:03:50.369125392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:03:50.622425505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:03:53.687324726Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-13T17:03:55.622783865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:03:58.042319171Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=warn ts=2026-05-13T17:04:00.622533778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:02.397769292Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-13T17:04:05.62244499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:06.752107396Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:04:10.62251112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:04:10.622521882Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:04:10.622551717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:04:10.622559927Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:04:11.107491554Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=debug ts=2026-05-13T17:04:14.794470709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:04:15.462292481Z caller=checkpoint.go:372 msg="writing series" size="445 kB"
level=debug ts=2026-05-13T17:04:15.462316548Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-13T17:04:15.622366201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:04:17.069189009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:38972"
level=info ts=2026-05-13T17:04:17.293262661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:04:17.303272252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:04:19.816630748Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-13T17:04:20.370456176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:04:20.622821049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:24.172858455Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T17:04:24.172900767Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=2
level=warn ts=2026-05-13T17:04:25.622225202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:28.526443186Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:04:30.62249772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:30.923821943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:04:32.881704869Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=warn ts=2026-05-13T17:04:35.621935604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:37.235971892Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=debug ts=2026-05-13T17:04:40.418326818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:04:40.594999846Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:04:40.599269164Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:04:40.622594176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:04:40.622613186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:04:40.622620791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:04:40.622679809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:41.590639093Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-13T17:04:45.622287846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:45.945924565Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=debug ts=2026-05-13T17:04:50.300862422Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T17:04:50.372178058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:04:50.622332979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:54.65554832Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T17:04:55.622570671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:04:59.010610327Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T17:04:59.010636515Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=8
level=warn ts=2026-05-13T17:05:00.621937242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:03.36512925Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-13T17:05:03.508630199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:05:03.518201444Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:05:05.622406017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:06.539171329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:05:06.539242405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:05:06.546294012Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:05:07.719953149Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T17:05:10.621990162Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:05:10.622029404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:05:10.622039653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:05:10.622063157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:12.074351081Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=warn ts=2026-05-13T17:05:15.622605584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:16.42957698Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
ts=2026-05-13T17:05:17.072709906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:53232"
level=debug ts=2026-05-13T17:05:18.353152157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:05:18.355218194Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T17:05:19.242086206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:05:19.25044446Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:05:20.373868349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:05:20.622407818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:20.783924225Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T17:05:25.138789329Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-13T17:05:25.622769141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:29.494007554Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=debug ts=2026-05-13T17:05:29.494034663Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-13T17:05:30.621959941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:33.849428633Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T17:05:35.622453972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:37.334049882Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:05:38.203775748Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T17:05:40.596761882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:05:40.598937902Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:05:40.622323147Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:05:40.622344714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:05:40.622354771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:05:40.622376059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:42.558712038Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-13T17:05:44.822847263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:44966"
level=warn ts=2026-05-13T17:05:45.62280632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:46.913246139Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
ts=2026-05-13T17:05:48.23263503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45758"
ts=2026-05-13T17:05:50.375183936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:05:50.622755599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:51.268202806Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-05-13T17:05:55.622183255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:05:55.623288367Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T17:05:57.404630861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:05:57.414546364Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:05:58.728814295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:05:58.736028324Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:05:59.977848408Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-13T17:06:00.622388003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:04.333258146Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-13T17:06:05.622781426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:08.688015865Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=debug ts=2026-05-13T17:06:10.512595371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:06:10.622550871Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:06:10.622593542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:06:10.622603828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:06:10.622632482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:13.042535444Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T17:06:13.042558406Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-13T17:06:15.622395312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:17.39694373Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-13T17:06:18.939337615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:06:20.376407016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:06:20.621959027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:21.752419108Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-13T17:06:25.622342694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:26.106604857Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-13T17:06:30.462340713Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-13T17:06:30.621942221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:34.816833447Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:06:35.622408189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:38.463751376Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:06:39.171171152Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T17:06:40.534010339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:06:40.543738105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:06:40.595559768Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:06:40.59865698Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:06:40.622007142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:06:40.622027714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:06:40.622037014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:06:40.622052513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:43.526831681Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=warn ts=2026-05-13T17:06:45.622305422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:47.881597087Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-13T17:06:50.378215454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=info ts=2026-05-13T17:06:50.556812078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:06:50.564734306Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:06:50.622866793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:52.236522853Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-13T17:06:55.62223405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:06:56.590482902Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-13T17:07:00.621963875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:00.945651629Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-13T17:07:05.301018198Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-13T17:07:05.301042386Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-13T17:07:05.62233844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:05.727673241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:07:07.799535826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:07:09.655297385Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T17:07:10.622409513Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:07:10.622427586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:10.622440459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:07:10.622479435Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:07:14.010490074Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=warn ts=2026-05-13T17:07:15.622606559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:18.365428303Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:07:20.379830412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:07:20.622396898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:22.720178824Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-13T17:07:25.622176796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:27.075019914Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T17:07:30.621964546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:31.429622053Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-13T17:07:32.123222458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:07:35.349582037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:07:35.359017829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:07:35.359040244Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:07:35.359063243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:07:35.365136711Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:07:35.622019208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:35.784908075Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-13T17:07:40.139023687Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-05-13T17:07:40.596077177Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:07:40.60004318Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:07:40.622535538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:07:40.62256726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:40.622581748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:07:40.622592944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:07:43.373124489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:07:44.494522297Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-13T17:07:45.622368946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:07:47.080071143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:53856"
level=debug ts=2026-05-13T17:07:48.84942713Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-13T17:07:49.508104727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:07:49.517232947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:07:50.380936965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:07:50.621967807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:53.204224366Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-13T17:07:54.339890737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:07:54.348186868Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:07:55.622665953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:07:57.5582183Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-13T17:07:57.558242957Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-13T17:08:00.622510426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:01.913777529Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-05-13T17:08:05.622406152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:06.271973656Z caller=checkpoint.go:372 msg="writing series" size="6.5 MB"
level=debug ts=2026-05-13T17:08:06.271997461Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.7 MB" series=2
level=debug ts=2026-05-13T17:08:07.212420158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:08:10.62243467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:08:10.622459368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:08:10.622485963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:08:10.622509028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:08:10.785703248Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-13T17:08:10.788111645Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.165577303s
ts=2026-05-13T17:08:14.831992264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:54030"
level=warn ts=2026-05-13T17:08:15.622477561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:08:20.382805029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:08:20.622022439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:21.465268543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:08:25.622283661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:08:28.468749709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:08:28.475503363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:08:30.386847027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:08:30.393798968Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:08:30.621993713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:35.537497498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:08:35.622773382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:40.596168614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:08:40.598241399Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:08:40.622605124Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:08:40.622632321Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:08:40.622647187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:08:40.622656283Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:08:40.622684756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:08:40.622740581Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-13T17:08:40.622771892Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:08:40.622817546Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=32.341µs success=true
level=debug ts=2026-05-13T17:08:40.623129188Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-13T17:08:44.085270938Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:08:45.622784417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:47.546940683Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
ts=2026-05-13T17:08:50.384652816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:08:50.622984531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:51.008493276Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=debug ts=2026-05-13T17:08:54.469897979Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-13T17:08:55.622240509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:08:57.931687867Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=warn ts=2026-05-13T17:09:00.622140425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:01.392688931Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-05-13T17:09:03.34518307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:09:04.854442824Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-13T17:09:05.622571293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:08.315576732Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=info ts=2026-05-13T17:09:10.62282646Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:09:10.622861191Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:09:10.622896555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:09:10.62293658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:11.777935278Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=debug ts=2026-05-13T17:09:15.239055618Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-13T17:09:15.602794023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:09:15.610032363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:09:15.622092867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:16.222661566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:09:16.224608657Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T17:09:18.700657076Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-13T17:09:19.506796072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:09:19.5133741Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:09:20.385805888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:09:20.622817241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:22.162478144Z caller=checkpoint.go:372 msg="writing series" size="557 kB"
level=debug ts=2026-05-13T17:09:22.162504134Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=warn ts=2026-05-13T17:09:25.622812632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:25.623920657Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T17:09:29.085664244Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:09:30.622809206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:30.851098549Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:09:32.555078884Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-13T17:09:32.555105362Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=3
level=warn ts=2026-05-13T17:09:35.62212408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:36.00801677Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-13T17:09:39.469641718Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=debug ts=2026-05-13T17:09:40.596179157Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:09:40.59824535Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:09:40.622705209Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:09:40.622727221Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T17:09:40.622717486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:09:40.622734881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:09:42.931740041Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
ts=2026-05-13T17:09:44.83632271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:42942"
level=warn ts=2026-05-13T17:09:45.62254556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:46.39256718Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-13T17:09:49.854637834Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
ts=2026-05-13T17:09:50.387181586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:09:50.622392397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:53.316450798Z caller=checkpoint.go:372 msg="writing series" size="436 kB"
level=debug ts=2026-05-13T17:09:53.316474888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T17:09:55.622135631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:09:56.777343081Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-13T17:09:58.289438867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:10:00.238855115Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-13T17:10:00.622503803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:10:01.252331587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:10:01.262710841Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:10:03.700955554Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-13T17:10:05.622080839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:07.162012937Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:10:10.573688143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:10:10.581310101Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:10:10.622660346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:10:10.622739271Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:10:10.622762842Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:10:10.622773512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:10:10.624747053Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T17:10:10.624759767Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-13T17:10:11.608086559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:10:14.085125192Z caller=checkpoint.go:372 msg="writing series" size="375 kB"
level=warn ts=2026-05-13T17:10:15.622267466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:17.546222738Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-13T17:10:20.388649025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:10:20.622330225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:21.00831878Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=debug ts=2026-05-13T17:10:24.469835773Z caller=checkpoint.go:372 msg="writing series" size="7.5 kB"
level=warn ts=2026-05-13T17:10:25.622306258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:25.878069776Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:10:27.931358245Z caller=checkpoint.go:372 msg="writing series" size="422 kB"
level=warn ts=2026-05-13T17:10:30.622350562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:31.393105807Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T17:10:31.393137086Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-13T17:10:34.85393787Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T17:10:35.622286575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:38.316070781Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=debug ts=2026-05-13T17:10:40.59635806Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:10:40.598465417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:10:40.621925368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:10:40.621951156Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:10:40.621961119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:10:40.621982506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:10:41.38535954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:10:41.392854884Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:10:41.777703203Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-13T17:10:44.838125375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:53130"
level=debug ts=2026-05-13T17:10:45.238719567Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-13T17:10:45.622442123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:10:48.260375123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34684"
level=debug ts=2026-05-13T17:10:48.700389698Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-13T17:10:50.389856128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:10:50.621957013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:51.65404331Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:10:52.162602505Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T17:10:55.622374626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:10:55.623424845Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-13T17:10:59.085286154Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-13T17:10:59.215630021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="49 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:10:59.224262475Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:11:00.621967091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:02.547202616Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T17:11:05.622550889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:06.008120423Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-13T17:11:09.470152643Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=debug ts=2026-05-13T17:11:09.47017517Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=11
level=info ts=2026-05-13T17:11:10.622700908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:11:10.622733791Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:11:10.622744891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:11:10.62277768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:11.284017126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:11:12.931837537Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
ts=2026-05-13T17:11:14.839948159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:46786"
level=warn ts=2026-05-13T17:11:15.625045253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:16.392852603Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-13T17:11:19.85389975Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-05-13T17:11:20.390723526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:11:20.62193836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:22.071366396Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:11:23.321119993Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-13T17:11:25.625343144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:26.777669028Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-13T17:11:27.045974774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:11:27.064403467Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:11:27.064433745Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:11:27.064462542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:11:27.071127758Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:11:30.239796694Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-13T17:11:30.624064191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:33.700580425Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:11:35.621958047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:37.161636507Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T17:11:38.050472836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:11:38.061298383Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:11:40.595049449Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:11:40.598257848Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:11:40.624285849Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:11:40.624326645Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:11:40.624337273Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:11:40.627963164Z caller=checkpoint.go:372 msg="writing series" size="674 kB"
level=debug ts=2026-05-13T17:11:40.627983635Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=warn ts=2026-05-13T17:11:40.628321321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:44.084940277Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T17:11:45.625045036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:11:46.15431604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:11:46.162124415Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:11:47.095329041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:60514"
level=debug ts=2026-05-13T17:11:47.547045519Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-13T17:11:48.593847149Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:11:49.556616962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:11:50.391994407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:11:50.621966074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:51.013039912Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-13T17:11:54.469760836Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-13T17:11:55.622809184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:11:57.931843266Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-13T17:12:00.622452041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:01.392442413Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T17:12:04.854805187Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T17:12:05.622424342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:08.316161622Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-13T17:12:10.621832224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:12:10.62185911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:12:10.621868669Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:12:10.621911283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:11.777673935Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T17:12:15.238837608Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-13T17:12:15.622623046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:12:17.976167533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:12:17.98512169Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:12:18.700867153Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:12:20.392791906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:12:20.622854042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:21.720857071Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:12:22.162026146Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=debug ts=2026-05-13T17:12:25.579293667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:12:25.622362067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:25.623401483Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-13T17:12:27.24085236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:12:27.248106705Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:12:29.085209014Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T17:12:30.622523729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:32.546289336Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-13T17:12:35.622077424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:36.008256446Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=debug ts=2026-05-13T17:12:39.469437505Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=debug ts=2026-05-13T17:12:39.469460139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=17
level=debug ts=2026-05-13T17:12:40.595948707Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:12:40.599155029Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:12:40.622631174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:12:40.622656286Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:12:40.622681367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:12:40.622664914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:42.931004207Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-13T17:12:45.62277538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:46.393235363Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-13T17:12:49.524005275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:12:49.854013716Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
ts=2026-05-13T17:12:50.39381823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:12:50.622905584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:53.315611537Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-13T17:12:55.622360377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:12:56.777795906Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-13T17:13:00.238586949Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-13T17:13:00.238611728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T17:13:00.622278518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:13:03.700302526Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:13:05.622934424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:13:06.985069944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:13:06.993408042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:13:07.165119185Z caller=checkpoint.go:372 msg="writing series" size="7.2 MB"
level=debug ts=2026-05-13T17:13:07.165140922Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.2 MB" series=2
level=info ts=2026-05-13T17:13:10.622664775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:13:10.622696949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:13:10.622705489Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:13:10.622701092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:13:10.634596973Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-13T17:13:10.641259839Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.018236087s
level=info ts=2026-05-13T17:13:13.916134575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:13:13.926031587Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:13:14.846786044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:34686"
level=warn ts=2026-05-13T17:13:15.62249622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:13:20.395108805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:13:20.622466871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:13:21.473656146Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:13:23.017606548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:13:23.019320912Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T17:13:25.622148124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:13:30.622716373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:13:35.622618491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:13:40.59668053Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:13:40.598794425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:13:40.622264602Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T17:13:40.622289044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:13:40.622290619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:13:40.622309973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:13:40.622321139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:13:40.622381124Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-13T17:13:40.622389507Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:13:40.622413909Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.657µs success=true
level=debug ts=2026-05-13T17:13:40.622682738Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-05-13T17:13:44.084659413Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-13T17:13:45.622519576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:13:45.727904565Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:13:47.546193312Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
ts=2026-05-13T17:13:50.396107316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:13:50.622018465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:13:51.008087263Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=debug ts=2026-05-13T17:13:54.469370584Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T17:13:55.622497781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:13:57.931355751Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=warn ts=2026-05-13T17:14:00.622664618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:01.392222548Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-13T17:14:02.0951922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:14:02.102177876Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:14:02.102198543Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:14:02.102221075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:14:02.107820528Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:14:02.345422865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:14:02.351634299Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:14:04.853982309Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-13T17:14:05.622231321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:08.315963171Z caller=checkpoint.go:372 msg="writing series" size="6.4 kB"
level=warn ts=2026-05-13T17:14:10.622470748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:14:10.622466463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:14:10.622528862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:14:10.622537346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:14:11.776831187Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-13T17:14:12.553512212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:14:12.560599564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:14:15.238322536Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-13T17:14:15.622367394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:14:17.104150104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:47374"
level=debug ts=2026-05-13T17:14:17.637563933Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:14:18.274674645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54544"
level=debug ts=2026-05-13T17:14:18.700353877Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T17:14:18.700376205Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
ts=2026-05-13T17:14:20.397189271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:14:20.622247814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:22.161476539Z caller=checkpoint.go:372 msg="writing series" size="700 kB"
level=debug ts=2026-05-13T17:14:22.855477301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:14:25.622581841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:25.623397214Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T17:14:25.623410398Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=debug ts=2026-05-13T17:14:29.084244954Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:14:30.622377632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:32.559630853Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-13T17:14:32.55965573Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=2
level=info ts=2026-05-13T17:14:33.518924894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:14:33.527966301Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:14:35.622731703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:36.007855941Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-13T17:14:39.469189522Z caller=checkpoint.go:372 msg="writing series" size="749 kB"
level=debug ts=2026-05-13T17:14:40.595656788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:14:40.598774511Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:14:40.622247183Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:14:40.622266693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:14:40.62227479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:14:40.622304308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:42.930960957Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=debug ts=2026-05-13T17:14:42.930989839Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-13T17:14:44.354678308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="165 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:14:44.364484039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:14:45.622286549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:46.392663801Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
ts=2026-05-13T17:14:47.105190171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:48728"
level=debug ts=2026-05-13T17:14:49.85397583Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=debug ts=2026-05-13T17:14:50.014572071Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:14:50.014641484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:14:50.020355401Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T17:14:50.398970687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:14:50.622283808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:53.315360043Z caller=checkpoint.go:372 msg="writing series" size="375 kB"
level=warn ts=2026-05-13T17:14:55.62210198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:14:56.777237832Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=debug ts=2026-05-13T17:15:00.238569947Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-13T17:15:00.622326683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:15:01.171990963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:15:01.181084252Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:15:03.699987943Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-13T17:15:04.463442786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:15:05.622445372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:07.162117669Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:15:10.622681142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:15:10.622710392Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:15:10.622719292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:15:10.622735769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:10.623755372Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T17:15:10.623773322Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=8
level=debug ts=2026-05-13T17:15:14.084913204Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=warn ts=2026-05-13T17:15:15.621997886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:15:16.232538197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:15:16.242064683Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:15:17.54633279Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-13T17:15:18.278621878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33902"
ts=2026-05-13T17:15:20.400577454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:15:20.622751059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:21.008342012Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=debug ts=2026-05-13T17:15:24.469893088Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T17:15:24.816995947Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:15:25.622677943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:27.93145841Z caller=checkpoint.go:372 msg="writing series" size="772 kB"
level=debug ts=2026-05-13T17:15:27.931478344Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-13T17:15:30.453588431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:15:30.461145856Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:15:30.622747764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:31.392061357Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-13T17:15:31.498315275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="189 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:15:31.50562405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:15:34.854349222Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-13T17:15:35.622206025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:38.315343724Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-13T17:15:40.595451731Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:15:40.598576212Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T17:15:40.599838562Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T17:15:40.599890783Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976324 n=1
level=debug ts=2026-05-13T17:15:40.599938013Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976324 n=1
level=info ts=2026-05-13T17:15:40.622087908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:15:40.622105919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T17:15:40.622098758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:15:40.622113476Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:15:41.776716868Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T17:15:45.239116053Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=warn ts=2026-05-13T17:15:45.62244697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:48.700129994Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
ts=2026-05-13T17:15:50.401729347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:15:50.622587138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:52.161174935Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-13T17:15:52.338749532Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:15:55.622325733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:15:55.623440443Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-13T17:15:59.084486801Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-13T17:16:00.622443453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:02.546483535Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-13T17:16:05.622606106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:05.993029969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:16:06.008077599Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-05-13T17:16:06.008105298Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-13T17:16:07.937028435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:16:07.944370017Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:16:09.46934597Z caller=checkpoint.go:372 msg="writing series" size="804 kB"
level=info ts=2026-05-13T17:16:10.62229513Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:16:10.622294481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:10.622323145Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:16:10.622333236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:16:10.714917305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:16:10.722437391Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:16:12.930695664Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-13T17:16:12.930718009Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-13T17:16:15.622281913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:16.392197941Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-13T17:16:19.854401889Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-05-13T17:16:20.403518845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=debug ts=2026-05-13T17:16:20.502711969Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:16:20.62193181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:16:21.952201495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="189 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:16:21.961374666Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:16:23.315686497Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=warn ts=2026-05-13T17:16:25.622007324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:26.777401285Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-13T17:16:30.238247345Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-13T17:16:30.622170335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:33.700101076Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-13T17:16:35.622455622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:37.161488238Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:16:40.596448644Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:16:40.598574516Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T17:16:40.622248838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:16:40.622069788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:16:40.622285902Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:16:40.622296295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:16:40.623571787Z caller=checkpoint.go:372 msg="writing series" size="722 kB"
level=debug ts=2026-05-13T17:16:40.623587799Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-13T17:16:44.084839421Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T17:16:45.622940946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:45.670139752Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:16:47.546610348Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
ts=2026-05-13T17:16:50.405398575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:16:50.622811718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:51.007382891Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-13T17:16:54.469293295Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-13T17:16:55.622345037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:16:57.931228548Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-13T17:17:00.304697768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:17:00.622783163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:01.392150559Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T17:17:04.853696925Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T17:17:05.622587604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:17:07.712783789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:17:07.721750277Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:17:08.315195452Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-13T17:17:08.589571412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:17:08.594295158Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:17:10.622394287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:17:10.622428265Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:17:10.622438601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:17:10.622434008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:11.776841006Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T17:17:12.765167081Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:17:15.238366114Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-13T17:17:15.622578384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:17:17.10316347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:17:17.109492733Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:17:18.70025372Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:17:20.407007551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:17:20.622754814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:22.161604176Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-13T17:17:25.622378447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:25.623475918Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-13T17:17:29.084289643Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T17:17:30.622505646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:32.546150291Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-13T17:17:32.546181744Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=15
level=warn ts=2026-05-13T17:17:35.622733923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:36.007848664Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=debug ts=2026-05-13T17:17:39.469717202Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=debug ts=2026-05-13T17:17:40.596632993Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:17:40.598747387Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:17:40.622315687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:17:40.622334677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:17:40.622341614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:17:40.622349031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:42.930357644Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-13T17:17:45.622246677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:46.392826875Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-13T17:17:48.644833639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:17:48.644907625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="515 kB" avg_comp="515 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:17:48.648183156Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:17:48.826681609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:17:48.835043621Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:17:48.836430634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:17:48.838107212Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T17:17:49.854095971Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-13T17:17:49.854119083Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-05-13T17:17:50.407836935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:17:50.622537847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:53.315727421Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-13T17:17:55.622543167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:17:56.777592318Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-13T17:18:00.238607649Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-13T17:18:00.622416417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:18:01.983153372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:18:01.989442894Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:18:02.854119433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:18:02.861857389Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:18:03.699801202Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:18:05.622522054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:18:05.949731478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:18:05.957992228Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:18:05.958012797Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:18:05.958040389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:18:05.965331995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:18:07.171252466Z caller=checkpoint.go:372 msg="writing series" size="7.6 MB"
level=debug ts=2026-05-13T17:18:07.171276044Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.0 MB" series=5
level=info ts=2026-05-13T17:18:10.622014432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:18:10.622048869Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:18:10.622057003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:18:10.622066823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:18:10.627415115Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-13T17:18:10.628566036Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005974966s
level=debug ts=2026-05-13T17:18:15.310143391Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:18:15.622791094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:18:16.216673401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:18:16.225688391Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:18:20.409485263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:18:20.622383607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:18:25.622855394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:18:30.622117048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:18:34.517162572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:18:34.523637937Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T17:18:35.622545203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:18:40.594962871Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:18:40.599094201Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:18:40.622658084Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:18:40.622679908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:18:40.622697735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:18:40.622705184Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:18:40.622720691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:18:40.660591423Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-13T17:18:40.66060847Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:18:40.660643567Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=29.074µs success=true
level=debug ts=2026-05-13T17:18:40.660920727Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-13T17:18:40.875295065Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:18:41.088091246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:18:41.096075773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:18:42.94485143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="236 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:18:42.95152018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:18:44.079016712Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=warn ts=2026-05-13T17:18:45.622828007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:18:46.735042166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:18:47.12204504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:55240"
level=debug ts=2026-05-13T17:18:47.497227734Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T17:18:49.484362221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:18:49.49089713Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T17:18:50.412065079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:18:50.622227324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:18:50.914625918Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T17:18:54.332370019Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T17:18:55.62272141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:18:57.750634952Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-13T17:19:00.622116553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:01.16775355Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-13T17:19:04.585824796Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-13T17:19:04.585848687Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-13T17:19:05.622680021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:06.992822994Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:19:08.002787548Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-13T17:19:10.622540861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:19:10.622588927Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:19:10.622598545Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:19:10.622609528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:11.421622358Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-05-13T17:19:14.838540572Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-13T17:19:15.622857291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:18.256283884Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
ts=2026-05-13T17:19:20.41380884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:19:20.622086498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:21.674935548Z caller=checkpoint.go:372 msg="writing series" size="794 kB"
level=debug ts=2026-05-13T17:19:21.674958619Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-05-13T17:19:24.111030833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:19:24.115647706Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:19:25.093766313Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-13T17:19:25.0937876Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=1
level=warn ts=2026-05-13T17:19:25.622548049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:19:25.825162104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:19:25.832449979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:19:28.509969565Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:19:30.622722107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:31.953766038Z caller=checkpoint.go:372 msg="writing series" size="31 MB"
level=debug ts=2026-05-13T17:19:31.953792885Z caller=checkpoint.go:384 msg="flushing series" totalSize="31 MB" series=2
level=info ts=2026-05-13T17:19:34.305920995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:19:34.314285712Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:19:35.345853773Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:19:35.622432078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:19:35.651536902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:19:35.65159997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T17:19:35.659380717Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:19:38.381722456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:19:38.762650992Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-13T17:19:40.595044362Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:19:40.599162745Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:19:40.621960595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:19:40.621987131Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:19:40.621997265Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:19:40.622017078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:19:41.939168282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="236 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:19:41.945440599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:19:42.180215385Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=debug ts=2026-05-13T17:19:42.180235557Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-05-13T17:19:45.598396257Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=warn ts=2026-05-13T17:19:45.622521184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:19:47.125955812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:60946"
ts=2026-05-13T17:19:48.296018784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:40890"
level=debug ts=2026-05-13T17:19:49.015865686Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-13T17:19:49.282479013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:19:49.289806814Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
ts=2026-05-13T17:19:50.415220183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:19:50.622289442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:52.433940788Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=debug ts=2026-05-13T17:19:52.433961964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-13T17:19:55.622167221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:19:55.8511135Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-13T17:19:59.269446477Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-13T17:20:00.623138969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:02.686968619Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T17:20:05.622867322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:06.104056048Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-13T17:20:07.986719687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:20:09.522220599Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:20:10.622799359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:20:10.622797486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:10.622831935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:20:10.62285237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:20:12.94174043Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-13T17:20:12.941762015Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=6
level=warn ts=2026-05-13T17:20:15.622024665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:16.357944086Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=info ts=2026-05-13T17:20:16.506323907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:20:16.515868015Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:20:17.126990355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:45816"
level=debug ts=2026-05-13T17:20:19.774965379Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-13T17:20:20.416600688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:20:20.622441693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:21.517574654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T17:20:21.87265338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:20:21.880117927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:20:21.880137263Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:20:21.880161024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:20:21.887289882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:20:22.364549795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:20:22.369938186Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:20:23.192846813Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-13T17:20:25.62220669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:26.610460723Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T17:20:26.901441533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:20:26.910615893Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:20:30.029621311Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T17:20:30.029646127Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=warn ts=2026-05-13T17:20:30.622035919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:20:32.510908557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:20:32.520577525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:20:33.4464414Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T17:20:35.205944839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:20:35.622127069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:36.864100045Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-13T17:20:38.530698824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:20:38.541061891Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:20:40.281518336Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
level=debug ts=2026-05-13T17:20:40.594960366Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:20:40.599071574Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:20:40.622650851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:20:40.622673545Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:20:40.622682882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:20:40.622687851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:43.699033027Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:20:45.622114914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:47.116748456Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
ts=2026-05-13T17:20:50.41818989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=debug ts=2026-05-13T17:20:50.534479483Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-13T17:20:50.622716611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:53.952986406Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T17:20:55.622489964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:20:57.370000854Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T17:21:00.62262414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:21:00.788233885Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-13T17:21:00.788262474Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-13T17:21:04.205411279Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-05-13T17:21:04.988386547Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:21:05.340163715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:21:05.350121526Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:21:05.622912802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:21:07.623162319Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-13T17:21:10.622446646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:21:10.622428011Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:21:10.622526679Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:21:10.622537334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:21:11.042643214Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T17:21:11.042673833Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-13T17:21:14.459630508Z caller=checkpoint.go:372 msg="writing series" size="374 kB"
ts=2026-05-13T17:21:14.872862715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:55738"
level=debug ts=2026-05-13T17:21:15.165237218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:21:15.622354745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:21:15.833327208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:21:15.839931569Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:21:17.129573911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:39604"
level=debug ts=2026-05-13T17:21:17.876817112Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-05-13T17:21:18.301727829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37288"
ts=2026-05-13T17:21:20.419458775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:21:20.622295434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:21:20.826972072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:21:20.835297158Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:21:21.294630442Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T17:21:21.62091141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:21:21.627398138Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:21:21.627428696Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:21:21.627461027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:21:21.633309291Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:21:24.712007763Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-13T17:21:25.622011005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:21:28.129870141Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T17:21:30.622455844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:21:31.332932997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:21:31.345299789Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:21:31.548001063Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-13T17:21:32.541208403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:21:32.547519196Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:21:34.964901679Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-13T17:21:35.622323203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:21:38.383231541Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:21:39.188003185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:21:40.595216157Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:21:40.598288579Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:21:40.621835924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:21:40.621908547Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:21:40.621920077Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:21:40.622005108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:21:41.801196429Z caller=checkpoint.go:372 msg="writing series" size="931 kB"
level=debug ts=2026-05-13T17:21:41.801225105Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=9
level=debug ts=2026-05-13T17:21:45.218189439Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T17:21:45.622066102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:21:48.635860475Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
ts=2026-05-13T17:21:50.420941584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:21:50.622455226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:21:51.042168894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:21:51.050275419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:21:52.054169957Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-13T17:21:53.472084541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:21:53.474087874Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T17:21:55.471820139Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-13T17:21:55.622384919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:21:58.085649017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:21:58.090718967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:21:58.889319108Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T17:22:00.622010845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:02.306483248Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-13T17:22:05.621922125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:05.724202704Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T17:22:09.10808862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:22:09.117304294Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:22:09.142457476Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-13T17:22:10.62243865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:22:10.622474972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:22:10.622487038Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:22:10.622503781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:12.560597523Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=debug ts=2026-05-13T17:22:14.571359356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:22:14.654966965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:22:14.659805922Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:22:14.875777106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:38392"
level=warn ts=2026-05-13T17:22:15.621963465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:15.977906374Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-13T17:22:19.395358303Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:22:20.422907464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:22:20.621971082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:22.813092858Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T17:22:25.622478518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:26.231397555Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-13T17:22:29.648561926Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-13T17:22:30.277430218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:22:30.290160418Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:22:30.622785083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:33.0667613Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-13T17:22:33.066784918Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=15
level=warn ts=2026-05-13T17:22:35.622710052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:36.48457246Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-13T17:22:39.902674781Z caller=checkpoint.go:372 msg="writing series" size="453 kB"
level=debug ts=2026-05-13T17:22:40.596191639Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:22:40.599312265Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:22:40.621894073Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:22:40.621949782Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:22:40.621972181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:22:40.621965938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:43.319684159Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-13T17:22:43.477110656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:22:43.485244521Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:22:44.877246766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:39160"
level=warn ts=2026-05-13T17:22:45.622542589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:46.004707383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:22:46.737912785Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-13T17:22:46.73793527Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T17:22:46.978177431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T17:22:47.181608013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:22:47.186284147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:22:50.155628673Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
ts=2026-05-13T17:22:50.42369052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:22:50.622835042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:22:53.57309121Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-13T17:22:55.622639578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:22:55.868928578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:22:55.877684707Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:22:56.990364767Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-13T17:22:58.516672297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:22:58.521201674Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:23:00.40867926Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-13T17:23:00.622470295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:23:03.825821485Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:23:05.62262922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:23:07.25142951Z caller=checkpoint.go:372 msg="writing series" size="8.4 MB"
level=debug ts=2026-05-13T17:23:07.251457968Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.1 MB" series=6
level=info ts=2026-05-13T17:23:10.513639307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:23:10.522534569Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:23:10.62283097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:23:10.622842804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:23:10.622866407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:23:10.622891164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:23:10.667395283Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-13T17:23:10.669159705Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00836625s
ts=2026-05-13T17:23:14.879200721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:49066"
level=warn ts=2026-05-13T17:23:15.62246899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:23:18.310215109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:55914"
ts=2026-05-13T17:23:20.42548052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:23:20.622613996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:23:20.763923031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:23:25.621934744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:23:30.622434123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:23:35.622745445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:23:36.341971067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:23:36.368606106Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:23:36.382002337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:23:36.390886485Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:23:37.172664601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:23:37.17737543Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:23:40.147146239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:23:40.150323668Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:23:40.598514008Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T17:23:40.598522838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:23:40.621981808Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T17:23:40.622041776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:23:40.622045476Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:23:40.622068079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:23:40.622077619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:23:40.622097603Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-13T17:23:40.622107532Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:23:40.622132216Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.748µs success=true
level=debug ts=2026-05-13T17:23:40.622316895Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-13T17:23:41.625192524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:23:43.997984334Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T17:23:45.621921448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:23:47.138454939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:59076"
level=debug ts=2026-05-13T17:23:47.372772182Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T17:23:48.312181119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:40892"
ts=2026-05-13T17:23:50.427118267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:23:50.622380704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:23:50.747909767Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T17:23:54.024426193Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:23:54.122907388Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T17:23:55.621953805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:23:57.49809495Z caller=checkpoint.go:372 msg="writing series" size="423 kB"
level=warn ts=2026-05-13T17:24:00.622651767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:00.872842651Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=info ts=2026-05-13T17:24:00.893088404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:24:00.900889506Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:24:04.247862956Z caller=checkpoint.go:372 msg="writing series" size="744 kB"
level=debug ts=2026-05-13T17:24:04.247913821Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=8
level=warn ts=2026-05-13T17:24:05.622467562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:07.62333252Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
level=info ts=2026-05-13T17:24:10.622052505Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:24:10.62208886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:24:10.622098951Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:24:10.622120963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:10.997931083Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:24:14.372566149Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-13T17:24:15.621965167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:17.747897201Z caller=checkpoint.go:372 msg="writing series" size="380 kB"
level=debug ts=2026-05-13T17:24:19.804268482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:24:20.428119564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:24:20.621946372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:21.123647519Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=debug ts=2026-05-13T17:24:21.123671065Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-05-13T17:24:23.019721347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:24:23.022949928Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:24:23.022970214Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:24:23.022994194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:24:23.025665922Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:24:24.130307181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:24:24.137954401Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:24:24.502274872Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:24:24.502298687Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=debug ts=2026-05-13T17:24:25.134368458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:24:25.621993608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:27.872667743Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:24:30.62288517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:31.281952991Z caller=checkpoint.go:372 msg="writing series" size="39 MB"
level=debug ts=2026-05-13T17:24:31.281981864Z caller=checkpoint.go:384 msg="flushing series" totalSize="39 MB" series=2
level=debug ts=2026-05-13T17:24:34.623120244Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-13T17:24:35.354163529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:24:35.364505118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:24:35.466936162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:24:35.472074049Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:24:35.622530278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:37.998510106Z caller=checkpoint.go:372 msg="writing series" size="936 kB"
level=debug ts=2026-05-13T17:24:40.595133164Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:24:40.599244911Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:24:40.622742581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:24:40.622765115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:24:40.622775331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:24:40.622770933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:41.373123403Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:24:41.373146144Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-13T17:24:42.063929145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:24:42.068010479Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:24:44.748124453Z caller=checkpoint.go:372 msg="writing series" size="441 kB"
level=warn ts=2026-05-13T17:24:45.622385247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:24:46.039910157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:24:46.048090543Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:24:46.048113532Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:24:46.048154173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:24:46.055754405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:24:47.141314319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:43932"
level=debug ts=2026-05-13T17:24:48.122662351Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
ts=2026-05-13T17:24:50.429612937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:24:50.622346841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:51.498395353Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=debug ts=2026-05-13T17:24:51.498415356Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-13T17:24:54.001789631Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:24:54.873416859Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=warn ts=2026-05-13T17:24:55.62270381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:24:58.247557867Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-13T17:25:00.622358749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:01.622561108Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-13T17:25:02.712654867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:25:02.718920913Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:25:03.658499096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:25:03.667197477Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:25:04.998034497Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T17:25:05.622523181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:08.372696251Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-13T17:25:08.970751367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T17:25:10.621944437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:25:10.621994904Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:25:10.622005853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:25:10.622039193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:11.750251273Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-13T17:25:11.750273769Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=6
level=debug ts=2026-05-13T17:25:15.12336551Z caller=checkpoint.go:372 msg="writing series" size="516 kB"
level=warn ts=2026-05-13T17:25:15.622565566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:25:16.669815681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:25:16.678181158Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
ts=2026-05-13T17:25:18.319031356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56344"
level=debug ts=2026-05-13T17:25:18.498141019Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-13T17:25:19.637827839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:25:20.431411021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:25:20.622526966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:21.87265697Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-13T17:25:25.247541768Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T17:25:25.622469102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:25:27.190002679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:25:27.201043273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:25:27.619471195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:25:27.62737103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:25:28.624430277Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T17:25:28.624451457Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=5
level=warn ts=2026-05-13T17:25:30.622378092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:31.997832386Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T17:25:34.187649814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:25:34.193504819Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:25:35.372545912Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-13T17:25:35.622335757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:38.747745191Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-13T17:25:40.595087387Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:25:40.599223879Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:25:40.622731322Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:25:40.622751138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:25:40.622758576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:25:40.62274813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:42.123702468Z caller=checkpoint.go:372 msg="writing series" size="580 kB"
level=debug ts=2026-05-13T17:25:45.497530566Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:25:45.622746982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:25:47.144772689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:48304"
level=debug ts=2026-05-13T17:25:48.873056235Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-05-13T17:25:48.873075841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-13T17:25:49.605652172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:25:49.607760085Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T17:25:50.433124488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=debug ts=2026-05-13T17:25:50.557991963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:25:50.622004037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:52.248195366Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-05-13T17:25:55.621930014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:25:55.623039084Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T17:25:56.958745579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:25:56.962741634Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:25:58.997938749Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T17:26:00.622438028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:02.372654077Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-13T17:26:05.622592415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:05.74799021Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-13T17:26:08.243422937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:26:08.255448234Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:26:08.255477764Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:26:08.255502834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:26:08.26231775Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:26:09.122699111Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T17:26:10.622542693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:26:10.622580784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:26:10.622590364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:26:10.622611185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:26:11.646440306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:26:11.653429433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:26:12.498806774Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T17:26:12.498830068Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=7
level=warn ts=2026-05-13T17:26:15.622152172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:15.873534504Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
ts=2026-05-13T17:26:18.322282164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60926"
level=debug ts=2026-05-13T17:26:19.028378274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:26:19.248049234Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
ts=2026-05-13T17:26:20.434803821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:26:20.622338331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:22.622360576Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T17:26:23.647908069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:26:23.65874514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:26:25.622457327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:25.997981149Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-13T17:26:26.814404902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:26:26.821619541Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:26:27.44065012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:26:27.446373714Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:26:29.37268941Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T17:26:30.622010572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:31.851009506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:26:32.747993027Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-13T17:26:35.622330992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:36.122576085Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-13T17:26:39.498138663Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:26:40.595399739Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:26:40.598526254Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T17:26:40.622020971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:26:40.622043128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:26:40.622071499Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:26:40.622083292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:26:42.873142881Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T17:26:42.873171353Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=9
level=warn ts=2026-05-13T17:26:45.622493167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:46.247614631Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-13T17:26:47.147999781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:52210"
ts=2026-05-13T17:26:48.324396162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44366"
level=debug ts=2026-05-13T17:26:49.622785912Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
ts=2026-05-13T17:26:50.435998287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:26:50.622290087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:51.835534019Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:26:52.997734172Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T17:26:54.662807656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:26:54.672959255Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:26:55.121556129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:26:55.12531269Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T17:26:55.622910349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:26:56.373094975Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-13T17:26:59.748355559Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T17:27:00.622019808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:03.123256865Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-13T17:27:03.842556846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:27:03.852817967Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:27:03.852903632Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:27:03.852959401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:27:03.863101597Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:27:05.62224008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:27:05.77600822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:27:05.783947564Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:27:06.497619233Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T17:27:09.641327453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:27:09.648068005Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:27:09.872741977Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-13T17:27:10.621827608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:27:10.621864604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:27:10.621892435Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:27:10.621985183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:13.24830463Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=warn ts=2026-05-13T17:27:15.62225221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:16.623285445Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-13T17:27:16.623350945Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T17:27:16.727359422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:27:16.739187419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:27:17.113663314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:27:17.149646985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:52256"
level=debug ts=2026-05-13T17:27:19.997902334Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:27:20.437737901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:27:20.622436045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:23.236399074Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:27:23.373384549Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T17:27:25.622633908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:26.747928323Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-13T17:27:30.123252477Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-13T17:27:30.622841449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:33.497700501Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-13T17:27:35.62241928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:36.87306075Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-13T17:27:38.553956729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:27:38.562973341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:27:39.546688911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:27:39.554124179Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:27:40.248609219Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=debug ts=2026-05-13T17:27:40.595735596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:27:40.598855176Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T17:27:40.622396938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:27:40.62240196Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:27:40.622424701Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:27:40.6224438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:27:43.622555731Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-13T17:27:45.622279709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:46.998053839Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-13T17:27:46.9980823Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-05-13T17:27:47.798220099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:27:47.801969895Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:27:50.372700644Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-13T17:27:50.439307678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:27:50.622424901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:27:52.96778962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:27:52.974495158Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:27:53.678699819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:27:53.748091318Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-13T17:27:53.847587598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:27:55.622411851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:27:57.123082537Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=info ts=2026-05-13T17:27:57.539132166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:27:57.544111243Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:27:57.544133099Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:27:57.544156878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:27:57.548406013Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:28:00.497694821Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=warn ts=2026-05-13T17:28:00.622820057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:28:03.872806734Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:28:05.622423914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:28:07.254960244Z caller=checkpoint.go:372 msg="writing series" size="9.2 MB"
level=debug ts=2026-05-13T17:28:07.254983134Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=6
level=info ts=2026-05-13T17:28:10.248743797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:28:10.257288548Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:28:10.622779186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:28:10.622815049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:28:10.622825543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:28:10.622850273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:28:10.964529622Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-13T17:28:10.966393078Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.344138204s
level=warn ts=2026-05-13T17:28:15.622364942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:28:17.151929104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:51562"
ts=2026-05-13T17:28:18.331038343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42038"
ts=2026-05-13T17:28:20.441193401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:28:20.622821277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:28:24.83392923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:28:24.833987741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T17:28:24.840608418Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:28:25.621953898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:28:28.64874581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:28:28.655736052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:28:30.41745085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:28:30.422284585Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:28:30.622133081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:28:35.622043336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:28:36.703834692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:28:36.710811291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:28:40.595233748Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:28:40.59828618Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:28:40.621733863Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:28:40.621917893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:28:40.621937832Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:28:40.621947969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:28:40.6219688Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=warn ts=2026-05-13T17:28:40.62197039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:28:40.621982646Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:28:40.622007262Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.445µs success=true
level=debug ts=2026-05-13T17:28:40.622232761Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-13T17:28:41.10411648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:28:41.111538077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:28:43.482477523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:28:43.998150492Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-05-13T17:28:44.899630384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:60674"
level=warn ts=2026-05-13T17:28:45.621992244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:28:47.203601268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:28:47.207190871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:28:47.372573457Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T17:28:50.042134022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:28:50.048292074Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
ts=2026-05-13T17:28:50.442519538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:28:50.622294351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:28:50.747686543Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T17:28:51.755427887Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:28:54.122674564Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:28:55.622072705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:28:57.497621418Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=warn ts=2026-05-13T17:29:00.622354729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:00.872448935Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-13T17:29:04.248092884Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=debug ts=2026-05-13T17:29:04.248114665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=8
level=warn ts=2026-05-13T17:29:05.62221681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:29:07.000923038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:29:07.007431746Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:29:07.623146735Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T17:29:10.62259846Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:29:10.622621138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:29:10.622629183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:29:10.62268362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:10.998346566Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:29:14.372518492Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-13T17:29:15.622043224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:16.063292957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:29:17.154250853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:35654"
level=debug ts=2026-05-13T17:29:17.747941861Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-13T17:29:17.747964778Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T17:29:18.000303592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:29:18.008014449Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T17:29:18.335222214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:40864"
ts=2026-05-13T17:29:20.443753886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:29:20.622540273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:21.122869353Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=info ts=2026-05-13T17:29:22.72039197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:29:22.726687417Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:29:24.50020539Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:29:24.500224696Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=2
level=warn ts=2026-05-13T17:29:25.622920283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:29:25.69422734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:29:25.70029554Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:29:27.873001503Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:29:30.622327736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:31.286799072Z caller=checkpoint.go:372 msg="writing series" size="42 MB"
level=debug ts=2026-05-13T17:29:31.286824142Z caller=checkpoint.go:384 msg="flushing series" totalSize="42 MB" series=2
level=info ts=2026-05-13T17:29:31.320324271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:29:31.327422847Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:29:34.62252993Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:29:35.622074131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:37.997738287Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T17:29:39.763136604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:29:39.764780951Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T17:29:40.594905562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:29:40.599056101Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:29:40.622566632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:29:40.622593803Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:29:40.622602116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:29:40.622621313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:41.372890585Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:29:41.372924849Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-13T17:29:44.748367086Z caller=checkpoint.go:372 msg="writing series" size="575 kB"
level=warn ts=2026-05-13T17:29:45.622101752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:29:45.891142992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:29:45.895579596Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:29:46.162277124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:29:48.123156824Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-05-13T17:29:50.444929278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:29:50.622787398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:51.49849264Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-05-13T17:29:51.498509583Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-05-13T17:29:54.873171596Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-13T17:29:55.622198425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:29:58.247910606Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-13T17:30:00.622389591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:30:00.790284145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:30:00.798016024Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:30:01.624490168Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T17:30:01.624512279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-13T17:30:04.997254471Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T17:30:05.622201625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:30:05.777159839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:30:05.781568884Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:30:08.372938502Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:30:10.622609287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:30:10.622637256Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:30:10.622645455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:30:10.622646061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:11.218839123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:30:11.748770757Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-13T17:30:11.748792866Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
ts=2026-05-13T17:30:14.905272892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:57412"
level=debug ts=2026-05-13T17:30:15.122542281Z caller=checkpoint.go:372 msg="writing series" size="393 kB"
level=info ts=2026-05-13T17:30:15.569513676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:30:15.580496235Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:30:15.622509867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:30:16.641524893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:30:16.648411205Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:30:17.157769633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:36780"
level=debug ts=2026-05-13T17:30:18.497995594Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-13T17:30:20.446489588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:30:20.622318609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:21.873164628Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T17:30:21.907277515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:30:21.914713662Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:30:25.247816536Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T17:30:25.62269978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:28.623749869Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T17:30:28.623777286Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=warn ts=2026-05-13T17:30:30.622302993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:31.997810259Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T17:30:35.283131867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:30:35.373255661Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-13T17:30:35.622295458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:38.747830816Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T17:30:39.246925279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:30:39.250342635Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:30:40.595169074Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:30:40.599284088Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T17:30:40.599474047Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T17:30:40.599519579Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976325 n=1
level=debug ts=2026-05-13T17:30:40.599558348Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976325 n=1
level=info ts=2026-05-13T17:30:40.622687562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:30:40.622706246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:30:40.622716837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:30:40.622767795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:41.749031526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:30:42.123360411Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
ts=2026-05-13T17:30:44.906368072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:52766"
level=debug ts=2026-05-13T17:30:45.497621421Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:30:45.622701986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:30:48.341566175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59652"
level=info ts=2026-05-13T17:30:48.785981211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:30:48.791162731Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:30:48.872322391Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-13T17:30:48.872345425Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-13T17:30:50.447724059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:30:50.62247833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:52.247823221Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-13T17:30:55.62253276Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T17:30:55.62253267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:30:58.997285617Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T17:30:59.767355574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:30:59.776361269Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:30:59.776383136Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:30:59.776414395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:30:59.782407961Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:31:00.35499165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:31:00.360014205Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:31:00.622376846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:02.372927382Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-13T17:31:05.622208491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:05.747583254Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-13T17:31:08.401262181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:31:08.411448903Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:31:09.122813463Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T17:31:10.622743371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:31:10.622768183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:10.622788079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:31:10.622795429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:31:12.083583589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:31:12.090183203Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:31:12.498270408Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T17:31:12.498292535Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=7
level=debug ts=2026-05-13T17:31:13.412092201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:31:14.907340725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:40552"
level=warn ts=2026-05-13T17:31:15.622245935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:15.873366583Z caller=checkpoint.go:372 msg="writing series" size="432 kB"
level=info ts=2026-05-13T17:31:16.761366259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:31:16.768109308Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:31:17.161490044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:35844"
level=debug ts=2026-05-13T17:31:19.24829813Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
ts=2026-05-13T17:31:20.448730729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:31:20.62261471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:22.623011873Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-13T17:31:25.622155483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:25.997509967Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-13T17:31:27.737324488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:31:27.742924385Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:31:27.74294784Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:31:27.742968801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:31:27.745707926Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:31:29.372256205Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-13T17:31:30.621995283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:32.747942976Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-13T17:31:35.622312073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:36.122440691Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-13T17:31:36.470434176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:31:37.733215791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:31:39.497824796Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T17:31:39.737627875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:31:39.746255451Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:31:40.594546188Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:31:40.598731927Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:31:40.622244325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:31:40.622263833Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:31:40.622271385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:31:40.622275512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:42.873682539Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T17:31:42.873704313Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=9
level=info ts=2026-05-13T17:31:45.071036285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:31:45.076541489Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:31:45.62196122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:46.247664531Z caller=checkpoint.go:372 msg="writing series" size="596 B"
level=info ts=2026-05-13T17:31:46.482444465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:31:46.485983117Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:31:48.345707973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46030"
level=debug ts=2026-05-13T17:31:49.623087541Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=info ts=2026-05-13T17:31:50.172381741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:31:50.177442281Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T17:31:50.449760981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:31:50.622942553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:31:51.370936193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:31:51.378437233Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:31:52.997561805Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T17:31:53.067608703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:31:53.074266286Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:31:55.623411868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:31:56.373171816Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-13T17:31:59.747356563Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T17:32:00.622790786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:03.122379781Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-13T17:32:05.622552139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:06.498316203Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T17:32:09.478995707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:32:09.484146349Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:32:09.87228647Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-13T17:32:10.621823865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:32:10.621861252Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:32:10.62187081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:32:10.622008382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:10.922136209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:32:13.248036244Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=warn ts=2026-05-13T17:32:15.622159425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:16.623298964Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-13T17:32:16.623323901Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
ts=2026-05-13T17:32:18.348252385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42612"
level=debug ts=2026-05-13T17:32:19.997619954Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:32:20.451042253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:32:20.622930372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:23.373228643Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T17:32:25.622350042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:26.747965193Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-13T17:32:27.344720532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:32:30.122219846Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T17:32:30.622377563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:33.498270887Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-13T17:32:35.622576852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:32:36.270692105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:32:36.278576173Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:32:36.54340648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:32:36.551777198Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:32:36.551806395Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:32:36.551841947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:32:36.558776881Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:32:36.872949772Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-13T17:32:38.736867136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:32:38.745782581Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:32:40.249257018Z caller=checkpoint.go:372 msg="writing series" size="623 kB"
level=debug ts=2026-05-13T17:32:40.595696761Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:32:40.598826571Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:32:40.622304355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:32:40.622335233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:32:40.622343598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:32:40.622349875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:41.394486985Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:32:43.623179224Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-13T17:32:43.623201644Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-05-13T17:32:44.913221758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:37900"
level=warn ts=2026-05-13T17:32:45.62206976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:46.997960776Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
ts=2026-05-13T17:32:47.165417025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:41978"
level=info ts=2026-05-13T17:32:47.167977234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:32:47.174682015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:32:48.409626001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:32:48.414743015Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:32:50.372749379Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
ts=2026-05-13T17:32:50.452465055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:32:50.622290486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:53.748483029Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-13T17:32:54.639904466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:32:54.652946534Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:32:55.622185801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:32:57.123514784Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=debug ts=2026-05-13T17:32:57.123566818Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T17:33:00.498495321Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-13T17:33:00.622449552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:33:03.872714359Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:33:05.622761499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:33:07.253869137Z caller=checkpoint.go:372 msg="writing series" size="9.9 MB"
level=debug ts=2026-05-13T17:33:07.254045805Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=3
level=info ts=2026-05-13T17:33:08.025403707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:33:08.031442806Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:33:10.622387611Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:33:10.622423713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:33:10.622434805Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:33:10.622643565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:33:10.633913468Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-13T17:33:10.637294449Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.015134497s
level=debug ts=2026-05-13T17:33:10.990643488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:33:13.654049887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:33:14.915043858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:60714"
level=warn ts=2026-05-13T17:33:15.622825393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:33:17.166869498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:59466"
ts=2026-05-13T17:33:20.453573004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:33:20.622551951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:33:20.733921707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:33:20.739698895Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:33:25.62253851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:33:27.471335408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:33:27.477919084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:33:30.622430451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:33:33.5852295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:33:33.595321949Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:33:35.622924874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:33:40.595208829Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:33:40.598251791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:33:40.621735191Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:33:40.621905911Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-13T17:33:40.621919535Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:33:40.621948863Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=23.669µs success=true
level=warn ts=2026-05-13T17:33:40.622033096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:33:40.622047719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:33:40.622070113Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:33:40.622079825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:33:40.622162647Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-13T17:33:43.997778015Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T17:33:44.390687342Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:33:44.390738428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:33:44.39543706Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T17:33:45.622537055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:33:46.109457528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:33:46.114542724Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:33:46.22594843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:33:46.232687958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:33:46.232710348Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:33:46.232738987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:33:46.238742451Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:33:47.372805419Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T17:33:48.354427647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53052"
ts=2026-05-13T17:33:50.454734103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:33:50.622181468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:33:50.747715219Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T17:33:54.123183317Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-13T17:33:55.622234696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:33:57.497979379Z caller=checkpoint.go:372 msg="writing series" size="443 kB"
level=debug ts=2026-05-13T17:33:58.606607551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:33:58.608137588Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T17:34:00.297509821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:34:00.300976824Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T17:34:00.622789198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:00.872729376Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=info ts=2026-05-13T17:34:02.367790061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:34:02.373843994Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:34:04.247744907Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=debug ts=2026-05-13T17:34:04.247767801Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=warn ts=2026-05-13T17:34:05.622927991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:34:06.381440529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:34:06.385470858Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:34:07.622925644Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-13T17:34:09.683848045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:34:09.689980396Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:34:10.622395325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:34:10.622419623Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:34:10.622427345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:34:10.622482136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:10.99822017Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:34:14.372680301Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-13T17:34:15.622529497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:16.670220425Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:34:16.96588387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:34:16.971782921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:34:16.971803055Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:34:16.971824459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:34:16.97717607Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:34:17.747730451Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-13T17:34:17.747753722Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-13T17:34:20.456339312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:34:20.622064058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:21.122734654Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=info ts=2026-05-13T17:34:21.786843052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:34:21.790357611Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:34:22.850565975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:34:22.858322592Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:34:24.499824808Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:34:24.499845723Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=2
level=warn ts=2026-05-13T17:34:25.622459862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:27.872666075Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:34:30.622483139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:31.280204745Z caller=checkpoint.go:372 msg="writing series" size="42 MB"
level=debug ts=2026-05-13T17:34:31.280234121Z caller=checkpoint.go:384 msg="flushing series" totalSize="42 MB" series=2
level=info ts=2026-05-13T17:34:32.198991066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:34:32.204152445Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:34:34.050141754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:34:34.056766648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:34:34.622516116Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:34:35.62277034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:34:35.953432407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:34:35.957838165Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:34:37.998432095Z caller=checkpoint.go:372 msg="writing series" size="912 kB"
level=debug ts=2026-05-13T17:34:40.595370062Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:34:40.598480107Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:34:40.621897275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:34:40.621915139Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:34:40.621922582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:34:40.621990044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:41.372837155Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:34:41.372859892Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-13T17:34:42.491239424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:34:42.491285973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:34:42.495041189Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:34:44.747740842Z caller=checkpoint.go:372 msg="writing series" size="596 kB"
level=warn ts=2026-05-13T17:34:45.622527956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:48.123212035Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
ts=2026-05-13T17:34:48.358625351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:39452"
ts=2026-05-13T17:34:50.457947841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:34:50.621980598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:34:51.497629387Z caller=checkpoint.go:372 msg="writing series" size="731 kB"
level=debug ts=2026-05-13T17:34:51.497650427Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-13T17:34:53.551791317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:34:53.558866623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:34:54.872711395Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T17:34:55.1563536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:34:55.161562216Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:34:55.622137727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:34:57.34083965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:34:57.344553061Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:34:58.247485907Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-13T17:34:58.418969513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T17:34:59.824073236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:34:59.830808538Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:35:00.622467956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:01.624164761Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T17:35:01.624190197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-13T17:35:03.908135066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:35:03.912000213Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:35:04.997962866Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T17:35:05.622441574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:06.725217742Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:35:08.372555194Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:35:10.434617514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:35:10.445607859Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:35:10.622384619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:35:10.62241265Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:35:10.622420328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:35:10.622430219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:11.749972715Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-13T17:35:11.749993393Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=debug ts=2026-05-13T17:35:15.12263417Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=warn ts=2026-05-13T17:35:15.622238049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:35:17.173973152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:39752"
level=debug ts=2026-05-13T17:35:18.497487801Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T17:35:19.170353307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:35:19.174611686Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T17:35:20.459777302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:35:20.622076529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:21.872986188Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-13T17:35:25.248065131Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T17:35:25.622671794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:35:27.290985191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:35:27.297442788Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:35:27.950658864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:35:27.955319977Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:35:27.955337462Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:35:27.955358099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:35:27.959609128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:35:28.624072884Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T17:35:28.62409322Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=warn ts=2026-05-13T17:35:30.622887049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:31.997592606Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T17:35:35.373191827Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:35:35.622316901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:36.442711201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:35:38.747961028Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-13T17:35:39.715309676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T17:35:39.778356646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:35:39.784194297Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:35:40.595147232Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:35:40.599283219Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:35:40.622739182Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:35:40.622761973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T17:35:40.622755511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:35:40.622769737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:35:42.123304005Z caller=checkpoint.go:372 msg="writing series" size="518 kB"
level=info ts=2026-05-13T17:35:42.593502856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:35:42.599917401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:35:44.10492711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:35:44.110093989Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:35:45.497603695Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:35:45.622455051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:48.872897651Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-13T17:35:48.87292182Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-13T17:35:49.941922176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:35:49.945556538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:35:50.461104048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:35:50.621993361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:52.247546706Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=warn ts=2026-05-13T17:35:55.622340776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:35:55.622365045Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T17:35:57.308394133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:35:57.31377276Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:35:58.997319981Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T17:36:00.622775937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:02.373244687Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-13T17:36:05.621932253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:05.747364881Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=info ts=2026-05-13T17:36:05.939262653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:36:05.943056655Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:36:09.122476409Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T17:36:09.219809511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:36:09.2277633Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:36:10.509586715Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:36:10.622080321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:36:10.622082046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:36:10.622106077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:36:10.622114157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:36:12.498739886Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T17:36:12.49876236Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=7
level=info ts=2026-05-13T17:36:14.777112573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:36:14.784348697Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:36:14.926414531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:51792"
level=warn ts=2026-05-13T17:36:15.622356848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:15.873097171Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=info ts=2026-05-13T17:36:17.109639967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:36:17.115633499Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:36:19.248447776Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
ts=2026-05-13T17:36:20.462527142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:36:20.622506886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:22.622431857Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-13T17:36:22.748814576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:36:25.622636476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:25.997563064Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-13T17:36:26.426301204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:36:26.432506379Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:36:29.372886622Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-13T17:36:30.622305178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:36:32.508983469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:36:32.512845904Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:36:32.74766019Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-13T17:36:34.373011125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:36:34.378562839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:36:35.622054723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:36.122234794Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-13T17:36:39.498110776Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:36:40.595009049Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:36:40.599127717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T17:36:40.622654228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:36:40.622673503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:36:40.622686087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:36:40.622693113Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:36:42.873127835Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T17:36:42.873149434Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=9
level=debug ts=2026-05-13T17:36:45.048613107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:36:45.622341854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:46.247679432Z caller=checkpoint.go:372 msg="writing series" size="643 B"
ts=2026-05-13T17:36:48.366511467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36056"
level=debug ts=2026-05-13T17:36:49.622630591Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
ts=2026-05-13T17:36:50.463701491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:36:50.622361078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:36:51.592533657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:36:51.597705655Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:36:52.997604232Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-13T17:36:55.575772279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:36:55.579852503Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T17:36:55.622379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:36:56.372930575Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-13T17:36:58.370862584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:36:58.376388721Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:36:59.747701408Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T17:36:59.909312757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:36:59.91818231Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:37:00.622116103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:03.122708028Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-13T17:37:05.622152441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:06.497991572Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T17:37:09.299303406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:37:09.30919124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:37:09.873024637Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-13T17:37:09.873056033Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T17:37:10.622353819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:37:10.622350438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:37:10.622404588Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:37:10.622414967Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:37:13.24772536Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
ts=2026-05-13T17:37:14.929649867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:50000"
level=debug ts=2026-05-13T17:37:15.274324679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:37:15.622430576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:16.622616199Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
ts=2026-05-13T17:37:18.368322585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43766"
level=debug ts=2026-05-13T17:37:19.997862262Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-13T17:37:20.049021493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:37:20.05744852Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:37:20.057477876Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:37:20.057521707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:37:20.064119421Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:37:20.251699857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:37:20.255335289Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:37:20.3296584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:37:20.465338369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:37:20.622502096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:23.37278079Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T17:37:25.622710903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:26.747638582Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-13T17:37:28.637342401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:37:28.645085519Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:37:30.12322528Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T17:37:30.622647823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:33.497719788Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-13T17:37:35.62291286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:36.872668433Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-13T17:37:38.957077527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:37:38.962524326Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:37:39.594552241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:37:39.60161189Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:37:39.838193546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:37:39.842355445Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:37:40.248405401Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=debug ts=2026-05-13T17:37:40.248426526Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-13T17:37:40.594774548Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:37:40.598897351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:37:40.622412866Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:37:40.622432346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:37:40.622440007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:37:40.62244609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:43.623092538Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=warn ts=2026-05-13T17:37:45.622558209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:46.99794771Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=info ts=2026-05-13T17:37:49.62073551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:37:49.625361624Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:37:50.373332607Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
ts=2026-05-13T17:37:50.466842778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:37:50.622282352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:52.853566816Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:37:53.747231174Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-05-13T17:37:53.747256028Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T17:37:55.622667349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:37:57.12328178Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=info ts=2026-05-13T17:37:58.019384983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:37:58.027784363Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:38:00.497708917Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-05-13T17:38:00.621957024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:02.280386942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:38:02.28824357Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:38:03.872489535Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:38:05.622446229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:38:07.258020033Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-13T17:38:07.25804602Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=4
level=info ts=2026-05-13T17:38:10.622572771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:38:10.622600782Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:38:10.622608419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:38:10.622692305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:10.631703798Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-13T17:38:10.634657342Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012567893s
level=info ts=2026-05-13T17:38:11.456667525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:38:11.462768556Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:38:14.013437682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:38:14.015099542Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T17:38:15.622968794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:17.16277385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:38:17.175038095Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:38:17.175062997Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:38:17.175087103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:38:17.177848094Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:38:17.184061379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:43342"
ts=2026-05-13T17:38:18.371870187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35244"
ts=2026-05-13T17:38:20.467703646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:38:20.622481274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:38:21.271940261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:38:25.622400354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:26.425754124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:38:26.432036216Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:38:30.432074016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:38:30.435857405Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T17:38:30.622311834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:32.997598289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:38:33.002538417Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:38:34.649674446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:38:34.654404128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:38:35.622847724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:38:40.595473705Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:38:40.598595762Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T17:38:40.622091051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:40.622106017Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:38:40.622110009Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:38:40.622129597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:38:40.622145373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:38:40.622248492Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-13T17:38:40.622258158Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:38:40.622280096Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.019µs success=true
level=debug ts=2026-05-13T17:38:40.622544743Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-13T17:38:43.997831406Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T17:38:45.622258601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:38:47.159112836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:38:47.163210228Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T17:38:47.185081545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:41058"
level=debug ts=2026-05-13T17:38:47.372963772Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T17:38:47.954368116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:38:47.960497673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:38:48.074645825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:38:50.469185967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:38:50.621933799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:38:50.748380217Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T17:38:50.802175839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:38:52.174471612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:38:52.1807773Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:38:54.12355465Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-13T17:38:55.622624822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:38:57.498569417Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=info ts=2026-05-13T17:38:57.615065233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:38:57.619643423Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:38:58.533324283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:38:58.540368541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:39:00.622269205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:00.872516048Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-13T17:39:04.248658817Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=debug ts=2026-05-13T17:39:04.248680197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=8
level=warn ts=2026-05-13T17:39:05.622364968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:39:06.914496535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:39:06.92143659Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:39:07.622571129Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T17:39:10.622473745Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:39:10.62251445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:10.622516056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:39:10.622534523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:39:10.998005877Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:39:14.372723476Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-13T17:39:15.622299293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:15.735720584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:39:17.748514429Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-13T17:39:17.748536218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T17:39:19.08732263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:39:19.093282165Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T17:39:20.47024808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:39:20.622322002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:21.122946182Z caller=checkpoint.go:372 msg="writing series" size="594 kB"
level=debug ts=2026-05-13T17:39:24.499825399Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:39:24.499844944Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=2
level=warn ts=2026-05-13T17:39:25.621916202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:27.872590845Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:39:30.622430847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:39:31.271639743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:39:31.281112237Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:39:31.29090683Z caller=checkpoint.go:372 msg="writing series" size="43 MB"
level=debug ts=2026-05-13T17:39:31.290926395Z caller=checkpoint.go:384 msg="flushing series" totalSize="43 MB" series=2
level=debug ts=2026-05-13T17:39:34.62351737Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:39:35.621948442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:37.998721257Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T17:39:40.595238944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:39:40.598367871Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:39:40.621811552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:39:40.621830282Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:39:40.621837622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:39:40.621977385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:41.37311108Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:39:41.37313474Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-13T17:39:41.442306122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:39:41.448042343Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:39:43.142184458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:39:43.654212315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:39:43.661568273Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:39:44.747639384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:39:44.747757724Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=warn ts=2026-05-13T17:39:45.622293012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:39:47.187622219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:53460"
level=info ts=2026-05-13T17:39:47.531924394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:39:47.53917313Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:39:47.539193112Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:39:47.539214432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:39:47.54586532Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:39:48.122724675Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
ts=2026-05-13T17:39:50.471608477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:39:50.622669759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:39:51.498533457Z caller=checkpoint.go:372 msg="writing series" size="812 kB"
level=debug ts=2026-05-13T17:39:51.498554139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-13T17:39:54.341019092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:39:54.346555271Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:39:54.872702507Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=warn ts=2026-05-13T17:39:55.622156646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:39:56.737007006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:39:56.740933556Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:39:57.774033846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:39:57.781511628Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:39:58.248144734Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-13T17:39:58.267208185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:39:58.274827088Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:40:00.622591845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:01.625031557Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T17:40:01.62505426Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-13T17:40:03.274163801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:40:03.278227837Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T17:40:03.278249291Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:40:03.278272327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:40:03.281492372Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:40:04.998360945Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T17:40:05.622936248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:07.845401166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:40:08.372722041Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T17:40:10.622313402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:40:10.622311883Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:40:10.622346198Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:40:10.622355554Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:40:11.752425655Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-13T17:40:11.752455221Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
level=debug ts=2026-05-13T17:40:15.122819764Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=warn ts=2026-05-13T17:40:15.622354219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:18.497902065Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T17:40:18.627521178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:40:18.63854271Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:40:19.816214243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:40:19.821330468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T17:40:20.473481308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:40:20.621968207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:40:21.195180193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:40:21.198845922Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:40:21.873325825Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=debug ts=2026-05-13T17:40:22.054470407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:40:25.247846258Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T17:40:25.622487551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:40:26.258015098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:40:26.262176285Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:40:28.623823296Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T17:40:28.623848498Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=5
level=info ts=2026-05-13T17:40:28.712349444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:40:28.720626735Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:40:28.720653366Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:40:28.720678126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:40:28.728121088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:40:30.622597332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:31.998546289Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T17:40:35.373443062Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:40:35.622771866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:38.747701913Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T17:40:38.804974739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:40:38.811266349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:40:38.811285622Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:40:38.811307794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:40:38.816097661Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:40:39.487997406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:40:40.595081265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:40:40.599200078Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:40:40.622719334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:40:40.622739671Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:40:40.62274757Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:40:40.622761323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:40:40.758173559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:40:40.765114723Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:40:41.801647654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:40:41.807847701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:40:42.122821697Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
ts=2026-05-13T17:40:44.941714716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:35256"
level=debug ts=2026-05-13T17:40:45.497705581Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:40:45.62226458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:40:46.548448036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:40:46.553486675Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T17:40:47.191316092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:37280"
ts=2026-05-13T17:40:48.381944255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41508"
level=debug ts=2026-05-13T17:40:48.872903659Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-13T17:40:48.872928573Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-13T17:40:50.47554144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:40:50.622394278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:52.248396961Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T17:40:54.697142646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:40:54.701935826Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T17:40:55.622779516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:40:55.622828098Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T17:40:58.186464091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:40:58.191860111Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:40:58.998158978Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T17:40:59.012253498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:40:59.019056281Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:41:00.622296502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:41:01.868987129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:41:01.873534676Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:41:02.373037232Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=warn ts=2026-05-13T17:41:05.62286226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:05.748276605Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-05-13T17:41:08.608237005Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:41:09.122651269Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T17:41:10.6228861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:41:10.622894558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:10.622923983Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:41:10.622936344Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:41:12.499363245Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T17:41:12.499391109Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=7
level=info ts=2026-05-13T17:41:14.515311779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:41:14.523067412Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:41:14.942686893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:53990"
level=warn ts=2026-05-13T17:41:15.622854791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:15.700077852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:41:15.873392084Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=info ts=2026-05-13T17:41:16.313107662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:41:16.317863478Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T17:41:16.317905358Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:41:16.317933953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:41:16.321704958Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T17:41:17.193235467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:33148"
level=debug ts=2026-05-13T17:41:19.24816688Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
ts=2026-05-13T17:41:20.47709454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:41:20.622058934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:22.623316206Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-13T17:41:25.622416505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:25.998095111Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=debug ts=2026-05-13T17:41:29.372986276Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-13T17:41:30.084347709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:41:30.092711873Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:41:30.622646481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:32.748027395Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-13T17:41:32.975066542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:41:32.981824295Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:41:35.346589479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:41:35.351519313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:41:35.622378165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:41:35.683532102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:41:35.686927974Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:41:36.123218116Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-13T17:41:37.392354636Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:41:37.392419583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:41:37.397584379Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:41:39.497990976Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T17:41:40.201241342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:41:40.204683641Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:41:40.595102926Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:41:40.599228076Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:41:40.622757578Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:41:40.622782683Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:41:40.622790357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:41:40.622816527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:42.873655236Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T17:41:42.873676739Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=9
level=warn ts=2026-05-13T17:41:45.622487806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:46.248266246Z caller=checkpoint.go:372 msg="writing series" size="625 B"
level=info ts=2026-05-13T17:41:46.358839327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:41:46.366214622Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
ts=2026-05-13T17:41:47.194677749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:38220"
level=info ts=2026-05-13T17:41:48.04432434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:41:48.049320203Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:41:49.623248729Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-13T17:41:50.47879406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:41:50.622837728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:41:52.998408769Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-13T17:41:55.622063313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:41:56.008240908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:41:56.013043645Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:41:56.373617565Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T17:41:57.463446352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:41:57.47007901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:41:59.747757298Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T17:42:00.62285417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:42:01.464543738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:42:01.469826676Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:42:01.469850454Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:42:01.469894012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:42:01.474942029Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:42:03.123005733Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T17:42:03.652895629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:42:05.622503127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:06.498528837Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T17:42:09.87230691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:42:09.872762229Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=error ts=2026-05-13T17:42:09.874011984Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T17:42:10.591771121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:42:10.596125492Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T17:42:10.622419663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:42:10.62243097Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:42:10.622460796Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:42:10.622472485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:42:13.163190389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:42:13.170257233Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:42:13.247581069Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
ts=2026-05-13T17:42:14.94608193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:33470"
level=warn ts=2026-05-13T17:42:15.622308711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:16.622925804Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-13T17:42:16.622954409Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
ts=2026-05-13T17:42:18.38841196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41636"
level=info ts=2026-05-13T17:42:18.643297017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:42:18.649852623Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:42:19.998474509Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-13T17:42:20.480189895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:42:20.622380475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:42:22.952643284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:42:22.957154829Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:42:23.373327316Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T17:42:25.052663811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:42:25.05817413Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:42:25.622850246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:26.748013038Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-13T17:42:28.066020774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:42:28.070203833Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:42:30.123508765Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T17:42:30.622631592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:33.498084248Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-13T17:42:35.549428059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:42:35.622529966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:36.873525796Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-13T17:42:40.026224513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:42:40.037352399Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:42:40.150600175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:42:40.156897652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:42:40.248894638Z caller=checkpoint.go:372 msg="writing series" size="692 kB"
level=debug ts=2026-05-13T17:42:40.594820845Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:42:40.598909271Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:42:40.622480616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:42:40.622481667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:40.622538233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:42:40.622549755Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:42:43.62286877Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-13T17:42:43.622925945Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-05-13T17:42:44.947529388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:40202"
level=warn ts=2026-05-13T17:42:45.621937316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:42:46.998511932Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-13T17:42:48.987287524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:42:50.373119404Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
ts=2026-05-13T17:42:50.481700069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:42:50.622417406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:42:53.732576199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:42:53.740143668Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:42:53.748277285Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-13T17:42:54.664539411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:42:54.669772332Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T17:42:55.622779522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:42:57.102186066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:42:57.109486366Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:42:57.122638209Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=debug ts=2026-05-13T17:42:57.122655058Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-13T17:43:00.498523616Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-13T17:43:00.622147271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:43:03.8727007Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-13T17:43:04.569950565Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:43:05.622402311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:43:07.258380043Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-13T17:43:07.258407359Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=info ts=2026-05-13T17:43:10.621920688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:43:10.621953856Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:43:10.62196266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:43:10.621961741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:43:10.629333332Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-13T17:43:10.632430709Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009984138s
level=info ts=2026-05-13T17:43:14.791938263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:43:14.800403092Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:43:15.622065987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:43:17.316285008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:43:17.320151474Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:43:20.482845798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=info ts=2026-05-13T17:43:20.580566962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:43:20.58557227Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:43:20.622780426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:43:21.247653657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:43:21.251060274Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:43:22.736227718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:43:22.740613204Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:43:25.622794987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:43:29.168285495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:43:30.005890534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:43:30.6226786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T17:43:35.622579142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:43:37.548591537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:43:37.556931368Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:43:40.595386435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:43:40.598515916Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:43:40.622017308Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:43:40.622052479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T17:43:40.622045278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:43:40.62204274Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:43:40.62206159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:43:40.622197617Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-13T17:43:40.622208864Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:43:40.622226414Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.653µs success=true
level=debug ts=2026-05-13T17:43:40.622478884Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-13T17:43:41.265095658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:43:41.269831014Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:43:43.997771063Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T17:43:44.739110863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:43:44.746099116Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:43:45.622725462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:43:47.372816155Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T17:43:48.395002877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59196"
level=info ts=2026-05-13T17:43:48.73943283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:43:48.744190314Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T17:43:50.484685732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:43:50.622669222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:43:50.74827774Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T17:43:54.1227889Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-13T17:43:55.622075053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:43:57.498576015Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=debug ts=2026-05-13T17:43:58.94658628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:44:00.622636709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:00.872690957Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-13T17:44:00.872715347Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=7
level=info ts=2026-05-13T17:44:01.37955573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:44:01.388433302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:44:01.388452038Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:44:01.388482771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:44:01.394902258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:44:02.905821932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:44:02.910898677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:44:04.24864459Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T17:44:05.19174804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:44:05.195183117Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T17:44:05.62290305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:07.622804242Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-13T17:44:10.622851732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:44:10.622854435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:44:10.622899238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:44:10.622909921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:44:10.997796611Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:44:10.997817371Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-05-13T17:44:14.373229044Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-13T17:44:15.39053994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:44:15.398927251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:44:15.621998461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:44:15.930433123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:44:15.935462541Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:44:17.338625403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:44:17.342780381Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:44:17.522907622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:44:17.7480524Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
ts=2026-05-13T17:44:20.486003311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:44:20.622554525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:21.123665593Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=debug ts=2026-05-13T17:44:21.123689522Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-05-13T17:44:24.501138994Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:44:24.501167797Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=warn ts=2026-05-13T17:44:25.621978297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:44:26.91516865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:44:26.920628724Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:44:27.873079525Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-13T17:44:30.622456375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:44:30.974573136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:44:30.982286848Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:44:31.277120967Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-13T17:44:31.277144942Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=2
level=debug ts=2026-05-13T17:44:33.372137243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:44:33.372193773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:44:33.37750181Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:44:34.622484957Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T17:44:35.622287299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:37.999052054Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T17:44:37.999075934Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-13T17:44:39.313406672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:44:39.319651213Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:44:39.925796662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:44:39.933700046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:44:40.595263941Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:44:40.598372846Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:44:40.621812775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:44:40.621831773Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:44:40.621853197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:44:40.621944799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:41.373528887Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=info ts=2026-05-13T17:44:42.544694816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:44:42.552809837Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:44:44.748262449Z caller=checkpoint.go:372 msg="writing series" size="492 kB"
level=debug ts=2026-05-13T17:44:44.748285559Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-05-13T17:44:44.954174534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:37694"
level=info ts=2026-05-13T17:44:45.000741767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:44:45.005173218Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=warn ts=2026-05-13T17:44:45.622916688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:44:47.204225488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:39768"
level=debug ts=2026-05-13T17:44:48.122718593Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
ts=2026-05-13T17:44:48.399281948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:55072"
ts=2026-05-13T17:44:50.487505008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:44:50.622383092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:51.498593808Z caller=checkpoint.go:372 msg="writing series" size="686 kB"
level=info ts=2026-05-13T17:44:54.744623628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:44:54.750663496Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:44:54.81201918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:44:54.815488147Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:44:54.815504513Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:44:54.815522963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:44:54.818577346Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:44:54.872831636Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-13T17:44:54.872845887Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-13T17:44:55.622085264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:44:58.248391593Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-13T17:44:58.298043295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:45:00.622593657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:01.624014849Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T17:45:01.624036618Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=debug ts=2026-05-13T17:45:04.602526837Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:45:04.997657129Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T17:45:05.019749048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:45:05.027213109Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:45:05.269807719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:45:05.273390591Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T17:45:05.622407086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:08.372903379Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:45:09.106691241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:45:09.110485064Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:45:10.62281615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:45:10.622847108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:10.622863152Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:45:10.622890039Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:45:11.751018183Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-13T17:45:11.751040025Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=debug ts=2026-05-13T17:45:15.123688479Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-05-13T17:45:15.622127672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:45:18.300563809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:45:18.308956688Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:45:18.497468726Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-13T17:45:20.488567684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:45:20.622309573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:21.87280225Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-13T17:45:23.245178869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:45:23.250717199Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=info ts=2026-05-13T17:45:25.21769852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:45:25.223073834Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:45:25.248186075Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T17:45:25.622541464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:45:26.355596309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:45:26.362971948Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:45:28.624555184Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T17:45:28.624577939Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=info ts=2026-05-13T17:45:29.853630307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:45:29.862673328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:45:30.62238169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:31.998325539Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T17:45:32.071468702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:45:32.075463159Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:45:35.372510534Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:45:35.621957519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:36.720426668Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:45:38.748370749Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T17:45:38.886837295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:45:38.890787403Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:45:40.594606304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:45:40.598711302Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T17:45:40.600022568Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T17:45:40.600061909Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976326 n=1
level=debug ts=2026-05-13T17:45:40.600101743Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976326 n=1
level=info ts=2026-05-13T17:45:40.622183641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:45:40.622203695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:45:40.6222115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:45:40.622238124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:45:41.094510491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:45:41.099296533Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:45:41.298329061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:45:41.299845981Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T17:45:42.123622175Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
ts=2026-05-13T17:45:44.958035947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:54626"
level=debug ts=2026-05-13T17:45:45.497617946Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T17:45:45.622320951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:45:47.730100176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:45:47.735090016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T17:45:48.403832804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38684"
level=debug ts=2026-05-13T17:45:48.873390853Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-05-13T17:45:48.873411449Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-13T17:45:50.489978403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:45:50.622667214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:45:52.248576983Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-13T17:45:55.622946415Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T17:45:55.622965655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:45:58.516086584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:45:58.526915063Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:45:58.526940837Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:45:58.526969555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:45:58.53386657Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:45:58.998454253Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T17:46:00.622487334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:02.372986662Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-13T17:46:05.622910265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:05.74837681Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-05-13T17:46:07.092006286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:46:07.095845778Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:46:07.915194025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:46:07.920163288Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:46:09.122858538Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T17:46:10.621917009Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:46:10.621925786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:10.62194435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:46:10.621953768Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:46:12.223669972Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:46:12.499338622Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T17:46:12.499360382Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=7
level=info ts=2026-05-13T17:46:14.13798393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:46:14.146642151Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:46:14.767628948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:46:14.772571644Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:46:15.204920379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:46:15.209185865Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=warn ts=2026-05-13T17:46:15.622295473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:15.873528266Z caller=checkpoint.go:372 msg="writing series" size="371 kB"
ts=2026-05-13T17:46:17.208290821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:33758"
level=debug ts=2026-05-13T17:46:19.248145176Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-05-13T17:46:20.491976815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:46:20.621943204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:46:21.815484821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:46:21.819743292Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:46:22.623156517Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T17:46:25.403578867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:46:25.410782865Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:46:25.51700919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.1 MB" avg_comp="2.3 MB" total_uncomp="79 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:46:25.524400323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:46:25.622551781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:25.998168287Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-13T17:46:29.373239127Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-13T17:46:30.621942703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:32.747654091Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-13T17:46:33.309499938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:46:33.314746394Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:46:35.622568167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:36.123108461Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-13T17:46:39.497699137Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:46:39.737039123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:46:40.59505235Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:46:40.599191652Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:46:40.622732662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:46:40.622755395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:46:40.622763702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:46:40.622762661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:42.874025579Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T17:46:42.874052981Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=9
level=info ts=2026-05-13T17:46:43.260395092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:46:43.265211611Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:46:45.433895728Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:46:45.62240969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:46:46.248423999Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=debug ts=2026-05-13T17:46:49.623557353Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-13T17:46:49.681615082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:46:49.687207686Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T17:46:50.494293339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:46:50.622587079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:46:52.276612653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:46:52.28331757Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:46:52.997680969Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-13T17:46:55.622376944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:46:56.141798066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:46:56.146702729Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:46:56.373388092Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T17:46:59.71565286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:46:59.719714034Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T17:46:59.719743709Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:46:59.719772461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:46:59.722743852Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:46:59.747960146Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T17:47:00.622901425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:47:03.122617402Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-13T17:47:04.224952396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:47:04.22830662Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:47:04.566660346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:47:04.571128204Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=warn ts=2026-05-13T17:47:05.621961846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:06.072594941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:47:06.080540378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:47:06.08056236Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:47:06.080590811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:47:06.087065329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:47:06.498399938Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T17:47:09.872762566Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-05-13T17:47:10.622195291Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:47:10.622226547Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:47:10.622234725Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:47:10.622269027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:11.882273095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:47:11.885784825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:47:13.248344534Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=debug ts=2026-05-13T17:47:14.154266076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:47:15.622401966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:47:16.622925287Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=debug ts=2026-05-13T17:47:16.622949916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-13T17:47:17.670675343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:47:17.676520776Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:47:17.676545084Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:47:17.676569946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:47:17.68096947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:47:19.997787589Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-13T17:47:20.496789899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:47:20.622434939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:21.598493747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:47:21.605616885Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:47:23.373224782Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T17:47:24.187066629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:47:24.193517873Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:47:25.622023467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:47:26.389347395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:47:26.748303354Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-13T17:47:29.694931806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:47:29.70070578Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:47:30.122925191Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T17:47:30.62213083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:32.730695804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:47:32.734222805Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:47:33.498080929Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-13T17:47:35.622314984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:47:36.872548477Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-13T17:47:37.432264155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:47:37.436782283Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T17:47:37.436805593Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:47:37.436829795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:47:37.441286942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:47:38.764937984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:47:38.769762789Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:47:38.769789225Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:47:38.769814781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:47:38.77417958Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:47:40.248604096Z caller=checkpoint.go:372 msg="writing series" size="716 kB"
level=debug ts=2026-05-13T17:47:40.595550342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:47:40.5986525Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:47:40.622091527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:47:40.622116909Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:47:40.622126983Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:47:40.62215577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:42.672420244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:47:42.677426988Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:47:42.974351017Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:47:43.623484222Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-13T17:47:43.623509076Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-13T17:47:44.519616321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:47:44.52681997Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
ts=2026-05-13T17:47:44.965603616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:42908"
level=warn ts=2026-05-13T17:47:45.622905509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:47:46.99814382Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=info ts=2026-05-13T17:47:48.008770472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:47:48.012756921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:47:50.373415174Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
ts=2026-05-13T17:47:50.498239116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:47:50.622005611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:53.262459339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:47:53.268158638Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:47:53.748238404Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-13T17:47:53.905775267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:47:53.913518136Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:47:55.622195854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:47:55.983857324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:47:55.988935382Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:47:57.123227448Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T17:47:57.123253336Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T17:48:00.325937368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:48:00.335787423Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:48:00.498551948Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=warn ts=2026-05-13T17:48:00.622637682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:00.96498253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:48:00.970918912Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:48:03.872777969Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:48:05.622717337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:48:07.263684917Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-13T17:48:07.263712074Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=3
level=info ts=2026-05-13T17:48:10.622554454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:48:10.62259404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:48:10.622609077Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:48:10.622613697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:10.630269691Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-13T17:48:10.63132187Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008952578s
level=debug ts=2026-05-13T17:48:14.510710222Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:48:15.622450075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:48:20.499867481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:48:20.622034303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:22.669557661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="380 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:48:22.677733983Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:48:23.552071985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:48:25.622334955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:26.220087967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:48:26.22463384Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:48:29.656000636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:48:29.659625352Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T17:48:30.621963512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:33.768740794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:48:33.780556392Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T17:48:35.622641177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:39.855779392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:48:39.863599191Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:48:40.237649204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:48:40.243021726Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:48:40.243059801Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:48:40.243083097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:48:40.247756559Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:48:40.594893706Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:48:40.598958887Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:48:40.623362082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:48:40.623399276Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:48:40.623411729Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:48:40.623524358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:40.623549381Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:48:40.623666592Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-13T17:48:40.623675127Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:48:40.623715696Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=34.932µs success=true
level=debug ts=2026-05-13T17:48:40.624140368Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-13T17:48:42.407687482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:48:42.414737883Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:48:43.917095689Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=debug ts=2026-05-13T17:48:43.990539325Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:48:44.968597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:50306"
level=warn ts=2026-05-13T17:48:45.622784877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:48:47.21021899Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T17:48:47.336406792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:48:47.340413554Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:48:48.416535206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43998"
level=info ts=2026-05-13T17:48:49.1250155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:48:49.130290652Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=info ts=2026-05-13T17:48:49.58551729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:48:49.59198256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:48:50.502136177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=debug ts=2026-05-13T17:48:50.502446983Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-13T17:48:50.622340183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:48:53.653558324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:48:53.658791737Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:48:53.795421898Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-13T17:48:55.622571636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:48:57.088380666Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=debug ts=2026-05-13T17:49:00.380153306Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-13T17:49:00.621937028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:03.673848044Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=debug ts=2026-05-13T17:49:03.673902782Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=8
level=warn ts=2026-05-13T17:49:05.622479308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:06.965919813Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-13T17:49:08.768437151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:49:09.734285543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="380 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:49:09.744354435Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:49:10.259296943Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=info ts=2026-05-13T17:49:10.622479895Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:49:10.622515129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:49:10.622526706Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:49:10.622546431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:13.551372459Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
ts=2026-05-13T17:49:14.97048101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:40302"
level=debug ts=2026-05-13T17:49:15.41533653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:49:15.622844221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:16.844060968Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-13T17:49:16.844087035Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T17:49:17.169608988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:49:17.174981609Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T17:49:17.845952003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:49:17.850038471Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:49:20.136852835Z caller=checkpoint.go:372 msg="writing series" size="790 kB"
ts=2026-05-13T17:49:20.504042212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:49:20.622850356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:49:21.72071576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:49:21.728903688Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:49:21.98561756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:49:21.989770894Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:49:23.43166922Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:49:23.431693743Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=2
level=warn ts=2026-05-13T17:49:25.622616218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:26.721916467Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-13T17:49:28.450013591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:49:28.458592862Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:49:28.494799218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:49:28.499532551Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:49:30.059132099Z caller=checkpoint.go:372 msg="writing series" size="49 MB"
level=debug ts=2026-05-13T17:49:30.059171779Z caller=checkpoint.go:384 msg="flushing series" totalSize="49 MB" series=2
level=warn ts=2026-05-13T17:49:30.62267042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:49:32.029803388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:49:32.035448766Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:49:33.307841469Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T17:49:35.62241599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:36.600900077Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T17:49:36.600930646Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-13T17:49:36.661240719Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:49:39.090007967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:49:39.095975525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:49:39.893841994Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:49:40.594985638Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:49:40.599096197Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:49:40.622657053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:49:40.622696614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T17:49:40.622697441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:49:40.622708587Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:49:43.186067773Z caller=checkpoint.go:372 msg="writing series" size="600 kB"
level=debug ts=2026-05-13T17:49:43.186092373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
ts=2026-05-13T17:49:44.972506052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:35496"
level=warn ts=2026-05-13T17:49:45.6223603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:46.478159243Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-05-13T17:49:48.421311975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42426"
level=info ts=2026-05-13T17:49:48.951509649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:49:48.959742216Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:49:49.772493939Z caller=checkpoint.go:372 msg="writing series" size="850 kB"
level=info ts=2026-05-13T17:49:50.414350278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:49:50.428310977Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:49:50.506149014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:49:50.622800895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:53.064145905Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-13T17:49:53.064172103Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-13T17:49:55.622476819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:49:56.356771069Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=info ts=2026-05-13T17:49:59.142682702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:49:59.148950699Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:49:59.650072496Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T17:49:59.650095493Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-13T17:50:00.622912212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:02.94209808Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T17:50:05.622639764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:06.234685133Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T17:50:07.958558207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:50:07.96409565Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:50:08.038605213Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:50:09.530608546Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:50:09.530636845Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T17:50:10.62260053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:50:10.622615447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:10.622631539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:50:10.622641858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:50:12.819773474Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
ts=2026-05-13T17:50:14.974332752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:47334"
level=warn ts=2026-05-13T17:50:15.622281889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:16.112139658Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-13T17:50:17.111608259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:50:17.113288452Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T17:50:17.22281162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:33046"
level=info ts=2026-05-13T17:50:18.86472253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:50:18.875295104Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:50:19.405018047Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
ts=2026-05-13T17:50:20.508321199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:50:20.622070658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:50:21.004411641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:50:21.008082541Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:50:21.115295753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:50:21.121138993Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:50:22.698277818Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T17:50:25.622761244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:25.991296094Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T17:50:25.991333012Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=info ts=2026-05-13T17:50:26.118651874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:50:26.122764857Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:50:28.139808554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:50:28.144490821Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:50:29.282961341Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T17:50:29.472479563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:50:29.479133668Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:50:30.622669248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:32.575734509Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-13T17:50:35.622566719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:35.868516356Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T17:50:36.812329998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:50:36.819354428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:50:36.819386341Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:50:36.819435921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:50:36.825599636Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:50:39.161870709Z caller=checkpoint.go:372 msg="writing series" size="697 kB"
level=debug ts=2026-05-13T17:50:39.200051429Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:50:39.839733105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:50:39.845795664Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:50:40.59467836Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:50:40.598791864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:50:40.622319475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:50:40.622341215Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:50:40.622351156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:50:40.622351349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:42.45371072Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-13T17:50:42.453731813Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-13T17:50:45.622247585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:45.746767667Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-13T17:50:49.039682984Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T17:50:50.509935104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:50:50.622263401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:50:51.824177772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:50:51.831974102Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:50:52.331342503Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T17:50:53.961378388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:50:53.966053582Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:50:54.517672073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:50:54.52456079Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:50:55.622770865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:50:55.624959081Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T17:50:58.91785491Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=warn ts=2026-05-13T17:51:00.622328036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:51:01.431094036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:51:01.434172512Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:51:02.209907391Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-13T17:51:05.502184353Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-13T17:51:05.50220606Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-13T17:51:05.622292378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:51:07.06475309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:51:07.068689835Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:51:07.068707615Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:51:07.068728258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:51:07.071707544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:51:08.796229885Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T17:51:08.796252545Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=info ts=2026-05-13T17:51:10.376469368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:51:10.383650054Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:51:10.442852664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:51:10.44694089Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:51:10.446961533Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:51:10.446983591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:51:10.450567275Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=info ts=2026-05-13T17:51:10.622641969Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:51:10.622634619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:51:10.622670187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:51:10.622679916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:51:12.08778507Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=debug ts=2026-05-13T17:51:12.670531072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:51:14.977300376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:36822"
level=debug ts=2026-05-13T17:51:15.380753175Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-05-13T17:51:15.409068138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:51:15.414231082Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:51:15.622334908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:51:15.842914103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:51:15.848295529Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T17:51:17.226022206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:33416"
level=debug ts=2026-05-13T17:51:17.617410116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:51:18.673617135Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
ts=2026-05-13T17:51:20.510981803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:51:20.622399043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:51:21.807856588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:51:21.815677577Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:51:21.965950994Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T17:51:23.95375336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:51:23.959418599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=info ts=2026-05-13T17:51:25.221526294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:51:25.224667386Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:51:25.258855455Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-05-13T17:51:25.258867779Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-13T17:51:25.622870414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:51:26.389137723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:51:26.395415379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:51:28.551526985Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-13T17:51:30.622345454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:51:31.843948241Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-13T17:51:35.137299045Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-13T17:51:35.6228013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:51:38.429684966Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:51:40.596968129Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:51:40.599070259Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:51:40.622588417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:51:40.622610914Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T17:51:40.622601669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:51:40.622618895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:51:41.722333856Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T17:51:41.722355992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=5
level=info ts=2026-05-13T17:51:41.897263686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:51:41.902479833Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T17:51:43.140057231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:51:43.144735814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:51:45.015014296Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T17:51:45.622525112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:51:46.001962068Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:51:47.435759478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:51:47.448289703Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:51:48.23045409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:51:48.234389691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:51:48.308113954Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
ts=2026-05-13T17:51:50.512806176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=info ts=2026-05-13T17:51:50.538900679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:51:50.543367335Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=warn ts=2026-05-13T17:51:50.62250328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:51:51.600694362Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T17:51:54.892735461Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-13T17:51:55.622009991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:51:58.185711264Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T17:52:00.622575019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:01.478887677Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-13T17:52:02.033621924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:52:02.042385208Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:52:02.566714748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:52:02.572027838Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:52:04.664049251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:52:04.670463753Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:52:04.7706905Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T17:52:05.622240972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:08.063302849Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-13T17:52:10.621832949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:52:10.62189192Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:52:10.621901673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:52:10.621923978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:52:10.647081348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:52:10.652747921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:52:11.049772296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:52:11.058024717Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:52:11.355955518Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=debug ts=2026-05-13T17:52:13.022666643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:52:14.649466102Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=debug ts=2026-05-13T17:52:14.649489118Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-05-13T17:52:15.622742007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:17.941652639Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:52:18.170386311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:52:20.514330674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:52:20.622902911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:21.23462612Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T17:52:24.526700506Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T17:52:25.622149103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:52:26.53592314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:52:26.541286657Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:52:27.819346711Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T17:52:30.622443382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:31.112139458Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T17:52:32.133706409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:52:32.138214259Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:52:34.16277423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:52:34.167274246Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:52:34.405248144Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-13T17:52:35.622855755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:37.697974528Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
level=debug ts=2026-05-13T17:52:40.595601796Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:52:40.598703716Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:52:40.622208719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:52:40.622232283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:52:40.622239383Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:52:40.622235281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:40.990757919Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-13T17:52:40.990778515Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-13T17:52:42.718623977Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:52:43.799984241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:52:43.806179093Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:52:44.283731949Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=warn ts=2026-05-13T17:52:45.622316821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:52:46.178449354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:52:46.182623623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:52:47.576178797Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T17:52:48.891180059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:52:48.897507532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T17:52:50.515361459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:52:50.622623225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:50.868527556Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=info ts=2026-05-13T17:52:53.387779642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:52:53.392377923Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:52:54.161523404Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-13T17:52:55.107007349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:52:55.11175696Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:52:55.622512717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:52:56.761493492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:52:57.45381676Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T17:52:57.453841056Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-13T17:53:00.622612812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:53:00.747193439Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=info ts=2026-05-13T17:53:01.920651943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:53:01.927319865Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:53:01.927340449Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:53:01.927364787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:53:01.933281785Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:53:04.039553781Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T17:53:05.62255315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:53:07.34678445Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-13T17:53:07.346808226Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=3
level=info ts=2026-05-13T17:53:08.917446741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:53:08.925347954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:53:09.493826948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:53:09.497564661Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:53:10.622646295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:53:10.622695885Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:53:10.622704086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:53:10.622677055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:10.639124086Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-13T17:53:10.640571303Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.016623007s
level=debug ts=2026-05-13T17:53:12.935760038Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:53:15.622786664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:53:20.516682477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:53:20.622703571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:21.471029642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:53:21.481149738Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:53:23.985593188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:53:23.990271688Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T17:53:25.622720661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:26.767834802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:53:26.771967815Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T17:53:26.771999701Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:53:26.772023009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:53:26.774522821Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T17:53:30.622549472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:31.702646993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:53:31.707007747Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:53:32.791651768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T17:53:35.622433964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:39.691485803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:53:39.694742459Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:53:40.594640456Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:53:40.59875497Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:53:40.622217306Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:53:40.622274247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:53:40.622307628Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:53:40.622315998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:53:40.622308592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:40.622366993Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-13T17:53:40.622389392Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:53:40.622430576Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=35.502µs success=true
level=debug ts=2026-05-13T17:53:40.622719353Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-13T17:53:41.720150163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:53:41.723763412Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:53:43.915925808Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-13T17:53:45.622492657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:46.908915874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:53:46.916085017Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:53:47.209053357Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-13T17:53:48.239969449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:53:49.080782959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:53:49.086286351Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:53:49.492367285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:53:49.498260763Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:53:49.498282214Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:53:49.498305599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:53:49.502711559Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:53:50.501272545Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-13T17:53:50.518531965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:53:50.622405091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:52.774303611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:53:52.780844217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:53:52.780865704Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:53:52.780915977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:53:52.787375085Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:53:53.794373391Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-13T17:53:55.622356856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:53:56.08130026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:53:56.089531756Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:53:57.086766558Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=debug ts=2026-05-13T17:53:57.086788189Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-13T17:54:00.379038562Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T17:54:00.622801193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:03.673019401Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=warn ts=2026-05-13T17:54:05.622135799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:54:06.398917746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:54:06.40495948Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:54:06.404978283Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:54:06.405000533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:54:06.409407763Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:54:06.964850072Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-13T17:54:08.808936535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:54:08.812846239Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T17:54:08.812892431Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:54:08.812923662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:54:08.816865423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:54:09.236947358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:54:09.24201865Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:54:09.374343958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:54:09.38056613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:54:10.257656079Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:54:10.25767795Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T17:54:10.622347289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T17:54:10.622352553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:10.622379128Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:54:10.622386878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:54:13.550485077Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-13T17:54:14.542819625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:54:14.544488883Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T17:54:14.987211342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:41472"
level=warn ts=2026-05-13T17:54:15.622609637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:16.842748972Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=info ts=2026-05-13T17:54:17.15830961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:54:17.162124081Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T17:54:17.235550343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:34704"
ts=2026-05-13T17:54:18.440491791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60072"
level=debug ts=2026-05-13T17:54:18.851417753Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:54:19.750104717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:54:19.754918407Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:54:20.135734857Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T17:54:20.135767388Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-13T17:54:20.519932828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:54:20.622129703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:54:22.638034948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:54:22.643135712Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:54:23.429928216Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:54:23.429969698Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-13T17:54:25.553998168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:54:25.557966892Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:54:25.622283637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:26.721121459Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-13T17:54:30.060095035Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-13T17:54:30.060119678Z caller=checkpoint.go:384 msg="flushing series" totalSize="52 MB" series=2
level=warn ts=2026-05-13T17:54:30.622663611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:33.306202534Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T17:54:35.622541925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:36.59938427Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=info ts=2026-05-13T17:54:38.46231415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:54:38.473094747Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:54:38.561315342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:54:38.570486314Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:54:38.85851946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:54:38.864007846Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T17:54:39.808735215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:54:39.813970748Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:54:39.891339202Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:54:39.891356477Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-13T17:54:40.367219476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:54:40.375313924Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:54:40.595105552Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:54:40.599232081Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:54:40.622742675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:54:40.622767247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:54:40.62277549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:54:40.622819356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:54:40.790182403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:54:40.794892642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:54:43.184524941Z caller=checkpoint.go:372 msg="writing series" size="664 kB"
level=debug ts=2026-05-13T17:54:44.98629889Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:54:44.986298137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:54:44.98927013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T17:54:45.287084102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:54:45.28983886Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T17:54:45.586616586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:54:45.589407376Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T17:54:45.622551546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:54:45.887072459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:54:45.889857288Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T17:54:46.186847668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:54:46.189697524Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T17:54:46.477460596Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
ts=2026-05-13T17:54:47.237176286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:50036"
level=info ts=2026-05-13T17:54:48.086385085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:54:48.091710323Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:54:49.770176078Z caller=checkpoint.go:372 msg="writing series" size="992 kB"
level=debug ts=2026-05-13T17:54:49.770200222Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
ts=2026-05-13T17:54:50.522241272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:54:50.622523783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:53.062444708Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=warn ts=2026-05-13T17:54:55.622302073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:54:56.354967073Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-05-13T17:54:58.004611069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:54:58.009064888Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:54:59.649158463Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T17:54:59.649180733Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-05-13T17:54:59.692717758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:54:59.697641318Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:55:00.622675216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:55:02.504441662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:55:02.507429207Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T17:55:02.940682792Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T17:55:03.591422413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:55:03.593967068Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T17:55:04.656777506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:55:04.659226436Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T17:55:04.784599842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:55:04.78719602Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T17:55:05.408729573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:55:05.411119663Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T17:55:05.622265902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:05.730940693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:55:06.232922572Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-13T17:55:09.529677555Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:55:09.529700665Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T17:55:10.621858142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:55:10.621901004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:55:10.621913482Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:55:10.621913948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:12.818313546Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=debug ts=2026-05-13T17:55:14.228398792Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T17:55:14.989615178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:36064"
level=warn ts=2026-05-13T17:55:15.621982028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:16.111494988Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
ts=2026-05-13T17:55:17.238028338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:36008"
level=debug ts=2026-05-13T17:55:19.404001237Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
ts=2026-05-13T17:55:20.523959991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=info ts=2026-05-13T17:55:20.612756443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:55:20.620147925Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=warn ts=2026-05-13T17:55:20.622429686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:55:21.147640365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:55:21.151021971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:55:22.696155248Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T17:55:23.429322459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:55:23.435740532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:55:24.790504192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:55:24.797848827Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T17:55:25.622308455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:25.989479528Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T17:55:25.98950181Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=5
level=info ts=2026-05-13T17:55:26.56499439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:55:26.567555303Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T17:55:29.247130074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:55:29.254736497Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:55:29.281812073Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T17:55:30.622329216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:32.5748458Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-13T17:55:33.77923174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:55:33.783711171Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:55:33.783732519Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:55:33.783754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:55:33.787736546Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:55:35.622203425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:35.867612201Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T17:55:37.665578899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:55:37.668058107Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T17:55:39.159811369Z caller=checkpoint.go:372 msg="writing series" size="780 kB"
level=debug ts=2026-05-13T17:55:39.159832478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T17:55:40.595723419Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:55:40.59884361Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:55:40.609109861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:55:40.613665766Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T17:55:40.621857833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:55:40.62189643Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:55:40.621905952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:55:40.621897469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:55:40.74237832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:55:40.74496534Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T17:55:41.359903779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:55:41.362693546Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T17:55:42.452898125Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-13T17:55:42.79219424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:55:42.794985149Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T17:55:43.679447235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:55:43.68452791Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:55:44.614797455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:55:44.618638646Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:55:45.622891016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:45.744844092Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-13T17:55:47.569292873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:55:48.252377192Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:55:49.037793133Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T17:55:50.165678547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:55:50.169182866Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T17:55:50.525770911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:55:50.627850811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:52.330537892Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T17:55:52.632739053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:55:52.637811268Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:55:55.622358307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:55:55.623475598Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T17:55:58.916011176Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=warn ts=2026-05-13T17:56:00.622472658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:01.896210484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:56:01.900567449Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:56:02.208733405Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-13T17:56:05.501260938Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-13T17:56:05.501283738Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T17:56:05.622295651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:56:08.795945756Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-13T17:56:08.795967613Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=warn ts=2026-05-13T17:56:10.622490576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:10.62248053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:56:10.622516426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:56:10.622533731Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:56:10.85903326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:56:10.866423891Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T17:56:11.659456066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:56:11.664599391Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=debug ts=2026-05-13T17:56:12.087114336Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=debug ts=2026-05-13T17:56:12.896032415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:56:14.068242991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:56:14.075915687Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:56:14.3225759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:56:14.328267578Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:56:15.379476438Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T17:56:15.62283303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:56:18.448258438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56398"
level=debug ts=2026-05-13T17:56:18.671772423Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-05-13T17:56:19.589925763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:56:19.597321738Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:56:20.527559265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:56:20.622436458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:21.889984717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:56:21.89269821Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T17:56:21.964805403Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-13T17:56:24.660814512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:56:25.257638422Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-05-13T17:56:25.606253658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:56:25.610164193Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:56:25.622250306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:27.405803566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:56:27.411504827Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:56:28.550425881Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-13T17:56:30.622853299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:30.992613856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:56:30.995182455Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T17:56:31.833524562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:56:31.83605577Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T17:56:31.843118257Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T17:56:32.235306688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:56:32.238403439Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T17:56:34.032210179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:56:34.035553837Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:56:35.135284144Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-13T17:56:35.135308913Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T17:56:35.622508419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:36.759497623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:56:36.76235374Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T17:56:38.428522042Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T17:56:40.595100548Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:56:40.599227631Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:56:40.622669788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:56:40.62270175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:56:40.622711005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:56:40.62281536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:56:40.788289828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:56:41.723214653Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T17:56:41.723231165Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=2
level=info ts=2026-05-13T17:56:41.958944513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:56:41.965964194Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:56:45.01332039Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T17:56:45.622140226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:56:47.660173618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:56:47.663838704Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:56:48.305783579Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
ts=2026-05-13T17:56:50.529166042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:56:50.622461284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:56:51.59857212Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T17:56:54.891509087Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-13T17:56:55.622734799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:56:58.18429337Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T17:56:58.548913064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:56:58.557779547Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:57:00.622408149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:57:01.477030623Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T17:57:04.770064164Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T17:57:05.622234507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:57:06.144356039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:57:06.149039252Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=info ts=2026-05-13T17:57:08.036796996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:57:08.041987973Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:57:08.062152435Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=debug ts=2026-05-13T17:57:08.062170502Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-13T17:57:08.945992144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:57:08.950109761Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:57:09.117701541Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:57:10.06440596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:57:10.070610712Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:57:10.07063051Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:57:10.070665235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:57:10.076579134Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:57:10.622011054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:57:10.622037588Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:57:10.622064228Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:57:10.622072897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T17:57:11.19692698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:57:11.199597798Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T17:57:11.354890532Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=debug ts=2026-05-13T17:57:14.648179784Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-13T17:57:15.622242338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:57:17.224440068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:57:17.230709252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T17:57:17.245262676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:49076"
level=debug ts=2026-05-13T17:57:17.940039981Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T17:57:18.401459019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:57:18.403801752Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T17:57:18.452389882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47170"
level=debug ts=2026-05-13T17:57:18.942057229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T17:57:20.530281526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:57:20.62193176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:57:20.632136015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:57:20.636227197Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:57:21.232705684Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T17:57:24.525685625Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T17:57:25.622525237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:57:26.654572768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:57:26.65747649Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T17:57:27.818816006Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T17:57:28.110624952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:57:28.113309699Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T17:57:29.711172893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:57:29.718017869Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T17:57:30.62222097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:57:30.772993403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:57:30.776022762Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T17:57:31.110607988Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T17:57:34.000478291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:57:34.00447657Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T17:57:34.403642634Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-13T17:57:35.162379161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:57:35.167206543Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:57:35.62207102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:57:36.076574894Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:57:36.332291464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:57:36.335559929Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:57:36.335579077Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:57:36.335602739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:57:36.338738426Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:57:37.696272197Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T17:57:37.69629443Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-13T17:57:40.594718616Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:57:40.598820993Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:57:40.622213852Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:57:40.622231112Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:57:40.622238718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:57:40.622280645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:57:40.989077843Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-13T17:57:42.14279327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:57:42.149560411Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:57:44.281617802Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=warn ts=2026-05-13T17:57:45.62248265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T17:57:47.246902038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:50984"
level=debug ts=2026-05-13T17:57:47.574332089Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T17:57:48.149699595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:57:48.155470664Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T17:57:48.454739907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53418"
level=info ts=2026-05-13T17:57:50.400710718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:57:50.404171815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:57:50.531710641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:57:50.622514484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:57:50.867273027Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-13T17:57:50.86729739Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T17:57:54.159746289Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-13T17:57:55.622868198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:57:57.452453331Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-13T17:57:57.515607619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:57:57.521235742Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T17:57:58.122797235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:57:58.126433735Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T17:57:58.352932672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:57:58.355510861Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T17:57:58.88552793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T17:57:58.887016335Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T17:57:58.991444098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:57:58.993663001Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T17:58:00.622866841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:58:00.745307261Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T17:58:01.444219368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T17:58:04.037776821Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T17:58:04.213411578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8bf1634729652b7b immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.7 MB" avg_uncomp="8.7 MB" synced=1 labels="{stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:58:04.218803937Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8bf1634729652b7b/19e224c92e0%3A19e226f13f4%3A99f871a7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zfte\"}"
level=error ts=2026-05-13T17:58:04.218823847Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:58:04.218845832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T17:58:04.221668986Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T17:58:05.622911336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:58:07.348608185Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T17:58:07.348647467Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=5
level=info ts=2026-05-13T17:58:08.482266985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:58:08.48875512Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T17:58:10.62259721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:58:10.622628319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:58:10.62263686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:58:10.62265437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:10.632083873Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-13T17:58:10.633524184Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.01092587s
level=info ts=2026-05-13T17:58:14.585830362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:58:14.589046098Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T17:58:14.633917146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T17:58:14.636783313Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T17:58:15.622289513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:18.024240137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:58:18.027334894Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T17:58:18.249939377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:58:18.259957666Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T17:58:18.785474984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:58:18.788640547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T17:58:20.532754518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:58:20.621999079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:20.647182806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:58:20.649748927Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T17:58:21.275166448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:58:21.280098867Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T17:58:25.62243674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:25.69075655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:58:25.693349181Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T17:58:29.569606794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:58:30.466565772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:58:30.472988607Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:58:30.622229219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:33.452829239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:58:33.457698899Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T17:58:35.622266596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:39.995784849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:58:39.998313191Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T17:58:40.595298415Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:58:40.598415134Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:58:40.621732807Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T17:58:40.621899203Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-13T17:58:40.621910142Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T17:58:40.621928018Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.312µs success=true
level=warn ts=2026-05-13T17:58:40.621934145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:40.621957696Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:58:40.621968452Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:58:40.621982119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T17:58:40.622202497Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-13T17:58:41.07485605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T17:58:43.915258849Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-13T17:58:45.622097097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:58:47.2078928Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T17:58:47.25087077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:58664"
level=info ts=2026-05-13T17:58:47.594528777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:58:47.598550375Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T17:58:48.458776091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49892"
level=debug ts=2026-05-13T17:58:50.500895014Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
ts=2026-05-13T17:58:50.534002888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T17:58:50.621916074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:50.945725577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T17:58:50.949759479Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T17:58:53.793914308Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T17:58:54.822869051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:58:54.857505078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:58:54.862100546Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T17:58:55.622262709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:58:56.446162335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:58:56.453329456Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:58:57.086375332Z caller=checkpoint.go:372 msg="writing series" size="424 kB"
level=debug ts=2026-05-13T17:58:57.086397077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-13T17:59:00.378578466Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T17:59:00.622369165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:59:03.67192837Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T17:59:05.443812277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T17:59:05.447445501Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T17:59:05.622400535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:06.245086474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:59:06.248687139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T17:59:06.964173643Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T17:59:07.044557746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:59:07.051560893Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:59:10.257676942Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T17:59:10.257705084Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T17:59:10.621904164Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:59:10.621937644Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:59:10.621949009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:59:10.621949559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:11.422762493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:59:11.425097392Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T17:59:13.004477225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:59:13.011225833Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:59:13.011245341Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T17:59:13.011267238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:59:13.017174127Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:59:13.549833191Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T17:59:13.654397427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:59:13.659109198Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T17:59:14.982201559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T17:59:14.985089891Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T17:59:15.622532936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:16.444138405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:59:16.448312408Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T17:59:16.842803767Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
ts=2026-05-13T17:59:18.460939999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38862"
level=debug ts=2026-05-13T17:59:20.135398509Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T17:59:20.135424445Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-13T17:59:20.535974185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T17:59:20.622793991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:23.345176997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T17:59:23.34981827Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T17:59:23.429909095Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T17:59:23.429937559Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=debug ts=2026-05-13T17:59:24.169425697Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T17:59:25.622044846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:26.569544115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T17:59:26.575719009Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T17:59:26.720392512Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T17:59:29.573311798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:59:29.580503394Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T17:59:30.057202904Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-13T17:59:30.057244291Z caller=checkpoint.go:384 msg="flushing series" totalSize="52 MB" series=2
level=warn ts=2026-05-13T17:59:30.621982824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:59:33.30551507Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T17:59:35.62291524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:59:36.599110548Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T17:59:36.599134163Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-13T17:59:37.013278757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T17:59:37.015687457Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T17:59:39.060169301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T17:59:39.430220617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T17:59:39.432725893Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T17:59:39.891370854Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T17:59:40.594940619Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T17:59:40.599042486Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T17:59:40.622373013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T17:59:40.622396626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T17:59:40.622408487Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T17:59:40.622435026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:42.455991138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T17:59:42.459910955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T17:59:43.18449024Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=debug ts=2026-05-13T17:59:43.184514549Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T17:59:44.362158034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T17:59:44.364648989Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T17:59:45.622269276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:59:46.476273163Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=debug ts=2026-05-13T17:59:48.374633841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T17:59:48.93333177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T17:59:48.941919088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T17:59:49.770019147Z caller=checkpoint.go:372 msg="writing series" size="901 kB"
level=debug ts=2026-05-13T17:59:49.770046097Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-05-13T17:59:50.537853258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T17:59:50.622688108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T17:59:51.938865673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T17:59:51.946359986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T17:59:53.062251889Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T17:59:53.278297673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T17:59:53.281669649Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T17:59:54.561306537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T17:59:54.569625903Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T17:59:55.622382593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T17:59:56.354225977Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-05-13T17:59:58.015238284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T17:59:58.020469051Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T17:59:58.163757882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T17:59:58.167028815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T17:59:58.972594204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T17:59:58.978568057Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T17:59:59.648337036Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T17:59:59.648358844Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-13T18:00:00.622567903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:00:01.046851701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:00:01.05041579Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T18:00:01.050435544Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:00:01.050458999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:00:01.053618061Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:00:02.939351734Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T18:00:05.622140606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:00:05.80274717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:00:05.805319022Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:00:06.232696943Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-13T18:00:09.526637939Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-13T18:00:09.526658914Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=warn ts=2026-05-13T18:00:10.622610683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:00:10.622632725Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:00:10.622658342Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:00:10.622672555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:00:12.267555953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:00:12.271611351Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:00:12.817593965Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=debug ts=2026-05-13T18:00:12.901732222Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:00:14.354932235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:00:14.358151643Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:00:15.622051351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:16.110625886Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T18:00:17.049332828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:00:17.055252094Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:00:19.403004991Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
ts=2026-05-13T18:00:20.539258425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:00:20.622301596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:22.696055773Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T18:00:25.622711936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:25.989059452Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T18:00:25.98907917Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=debug ts=2026-05-13T18:00:29.281192484Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T18:00:30.622792553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:32.574035811Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-13T18:00:32.868548422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:00:34.256799444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:00:34.260327794Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:00:34.858437103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:00:34.861621761Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T18:00:35.48752812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:00:35.489866267Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:00:35.622766041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:35.8664601Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T18:00:37.563101187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:00:37.569993642Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:00:37.570017058Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:00:37.570042509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:00:37.576669713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:00:38.541641127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:00:38.54857502Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:00:38.950782358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:00:38.953485261Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:00:39.159373977Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=info ts=2026-05-13T18:00:40.383755867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:00:40.38813765Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T18:00:40.38815771Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:00:40.388180366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:00:40.39127892Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T18:00:40.594957102Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:00:40.599062849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T18:00:40.599270403Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T18:00:40.599310158Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976327 n=1
level=debug ts=2026-05-13T18:00:40.599351898Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976327 n=1
level=info ts=2026-05-13T18:00:40.622428508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:00:40.622426269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:40.622449393Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:00:40.622456648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:00:42.451737953Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-13T18:00:42.451759864Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-13T18:00:42.84508062Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:00:45.622051999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:45.744662304Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-13T18:00:47.538841187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:00:47.541460106Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:00:48.182271506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:00:48.186544347Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:00:48.186565129Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:00:48.186586454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:00:48.190335876Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:00:48.619637881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:00:48.622026293Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:00:48.828613236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:00:48.835899835Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:00:49.037612197Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:00:49.946024465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:00:49.9531133Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T18:00:50.540442569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:00:50.622239923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:00:52.006432206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:00:52.012559552Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:00:52.012580532Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:00:52.012602316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:00:52.017409558Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:00:52.207992566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:00:52.211435272Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:00:52.329641574Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T18:00:54.692259996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:00:54.696837184Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T18:00:55.622191994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:00:55.622264572Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T18:00:58.668609809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:00:58.672165938Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:00:58.915210654Z caller=checkpoint.go:372 msg="writing series" size="293 kB"
level=warn ts=2026-05-13T18:01:00.622419608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:01:02.208660523Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-13T18:01:05.301244701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:01:05.306902092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:01:05.50069605Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-13T18:01:05.500718374Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-13T18:01:05.622598023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:01:06.093776687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:01:06.0985788Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:01:07.440854599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:01:07.769318637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:01:07.774368705Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:01:08.794847416Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-13T18:01:08.794869892Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=debug ts=2026-05-13T18:01:09.057620252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:01:10.622456562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:01:10.62247897Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:01:10.622488664Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:01:10.622514535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:01:12.086736319Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=debug ts=2026-05-13T18:01:15.379500716Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T18:01:15.622803694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:01:16.742624755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:01:16.745913991Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T18:01:18.468617861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58570"
level=debug ts=2026-05-13T18:01:18.671491777Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
ts=2026-05-13T18:01:20.542524578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:01:20.622523266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:01:20.816288763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:01:20.819654984Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:01:20.819673373Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:01:20.819696166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:01:20.82178051Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:01:21.124897476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:01:21.127486701Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:01:21.964553984Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-13T18:01:24.637556389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:01:24.64470765Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:01:24.779002305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:01:24.782999414Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T18:01:25.256378153Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-13T18:01:25.622557675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:01:28.549080075Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-13T18:01:28.549102267Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-13T18:01:30.27959451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:01:30.284194782Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:01:30.284218287Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:01:30.284245214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:01:30.286463147Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:01:30.6223594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:01:31.84251809Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T18:01:33.270634737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:01:33.273648245Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:01:33.617863583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:01:33.620788705Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:01:33.620808567Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:01:33.620829042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:01:33.6227437Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:01:35.13522161Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-13T18:01:35.62244173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:01:36.217467075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:01:36.226060907Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:01:36.436645738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:01:36.441228773Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:01:38.427870145Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-13T18:01:40.59538577Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:01:40.59851335Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:01:40.621817919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:01:40.621840694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:01:40.621856218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:01:40.621900767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:01:41.259459339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:01:41.721149359Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T18:01:41.721170303Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=info ts=2026-05-13T18:01:42.019610304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:01:42.022971014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:01:42.238488022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:01:42.240995894Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:01:42.24101178Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:01:42.241029151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:01:42.242893151Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:01:43.281786472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:01:43.284384267Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:01:43.811020508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:01:43.816768013Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T18:01:45.009160002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:48362"
level=debug ts=2026-05-13T18:01:45.012867402Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T18:01:45.622886774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:01:47.259028373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:01:47.266868987Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:01:48.306134916Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
ts=2026-05-13T18:01:50.544260693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:01:50.622661209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:01:51.598702474Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-13T18:01:53.415010908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:01:53.418437218Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:01:53.765238602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:01:53.76826193Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:01:54.890559028Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-13T18:01:55.622099102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:01:56.39322056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:01:56.396201908Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:01:57.628995862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:01:57.630691119Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T18:01:58.183462945Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T18:01:58.346796987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:01:58.349699228Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:01:58.349728819Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:01:58.34975401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:01:58.351928274Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:01:58.928552465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:01:58.932573962Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:01:59.170154296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:01:59.175918836Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:01:59.175968893Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:01:59.176002672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:01:59.181242458Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T18:02:00.622837357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:02:01.476051733Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T18:02:04.769532534Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T18:02:05.622026274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:07.580725505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:02:07.593755776Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:02:08.06166469Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-13T18:02:09.564545145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:02:09.571179452Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:02:09.818316449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:02:09.825078341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:02:10.622433043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:02:10.622471174Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T18:02:10.622462186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:10.622479688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:02:11.199722995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:02:11.353968924Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-13T18:02:14.575943432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:02:14.579851241Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T18:02:14.647071312Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-13T18:02:14.647091687Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-13T18:02:15.622764449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:16.872369375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:02:16.874972542Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:02:17.940040636Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T18:02:19.671997482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:02:19.674458445Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T18:02:20.546956183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:02:20.622057104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:02:21.232178714Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-13T18:02:23.053631358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:02:23.061355275Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:02:24.524549159Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T18:02:25.622635703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:26.850620103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:02:26.854061036Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:02:27.818267439Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T18:02:29.096981719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:02:29.099516339Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:02:30.622757972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:30.746114785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:02:30.751083881Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:02:30.902155386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:02:30.907016307Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:02:31.110750044Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T18:02:34.38452736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:02:34.387710406Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:02:34.402807047Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-13T18:02:35.622151957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:37.006520696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:02:37.011551797Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:02:37.69608382Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-13T18:02:40.594534669Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:02:40.598676242Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:02:40.622004947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:02:40.622017256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:02:40.62203855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:02:40.622047011Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:02:40.988251061Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=debug ts=2026-05-13T18:02:40.98827703Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-13T18:02:41.072405285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:02:41.079129046Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:02:42.258421228Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:02:43.995868603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:02:44.281830285Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=info ts=2026-05-13T18:02:45.479704929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:02:45.483170712Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:02:45.621944467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:02:47.574302976Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T18:02:49.999984056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:02:50.002851679Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T18:02:50.548624373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:02:50.622420838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:02:50.866648393Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-13T18:02:52.207428513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:02:52.210853669Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:02:54.15933194Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-13T18:02:55.622657717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:02:57.452216949Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T18:02:57.452243662Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T18:02:59.463832912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:02:59.471897531Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:03:00.622503201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:03:00.74508419Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=debug ts=2026-05-13T18:03:04.037028193Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-13T18:03:05.62240464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:03:06.022674642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:03:06.026361159Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:03:07.344340984Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T18:03:07.344368679Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-13T18:03:07.649437237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:03:07.652498824Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:03:08.304354638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:03:08.310064619Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T18:03:10.622919508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:03:10.622936308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:03:10.62296001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:03:10.622973068Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:03:10.639169554Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-13T18:03:10.640409285Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.018355244s
level=info ts=2026-05-13T18:03:12.694460949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:03:12.697489747Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:03:15.622543364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:03:17.180613436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:03:18.904021154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:03:18.909958179Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T18:03:20.55078126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:03:20.622446829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:03:21.056948436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:03:21.064484196Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:03:21.980680524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:03:21.985426977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:03:24.364996994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:03:24.369934511Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:03:25.040779121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:03:25.043994138Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:03:25.622466843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:03:28.45817743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:03:28.466384564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T18:03:30.622552144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:03:32.939083247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:03:33.872389805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:03:33.875858671Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:03:34.244021638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:03:34.249451889Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T18:03:35.622146268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:03:38.965998803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:03:38.970518992Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:03:39.463980089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:03:39.466373901Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:03:40.595265789Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:03:40.598418081Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:03:40.621723824Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T18:03:40.621904696Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-13T18:03:40.621915763Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T18:03:40.621939689Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.722µs success=true
level=warn ts=2026-05-13T18:03:40.62194205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:03:40.622005117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:03:40.622018444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:03:40.622027034Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:03:40.622233924Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-13T18:03:43.915724732Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=warn ts=2026-05-13T18:03:45.622486205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:03:46.416826509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:03:46.423547685Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:03:47.207886906Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T18:03:47.267709549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:54516"
level=debug ts=2026-05-13T18:03:50.501356363Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
ts=2026-05-13T18:03:50.553020128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:03:50.622627051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:03:51.323027496Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:03:53.793588141Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-13T18:03:55.622776537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:03:57.086513908Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=debug ts=2026-05-13T18:03:57.086539218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-13T18:04:00.379019643Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T18:04:00.621961116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:04:01.630122428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:04:01.635308655Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:04:03.672050733Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T18:04:04.150528383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:04:04.155509814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T18:04:04.587814831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:04:04.590551336Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T18:04:05.311279155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:04:05.315596717Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:04:05.315618013Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:04:05.315642548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:04:05.318925286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:04:05.622860176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:06.964092204Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-13T18:04:07.574905528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:04:07.577694222Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:04:08.06292105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:04:08.068076648Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:04:09.748729786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:04:09.755460699Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:04:10.257560269Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T18:04:10.257587814Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T18:04:10.622511599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:04:10.622544673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:04:10.622552888Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:04:10.622562465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:04:12.016825348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:04:12.021548835Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:04:13.549932229Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-13T18:04:15.017692272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:36126"
level=info ts=2026-05-13T18:04:15.468290088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:04:15.475500311Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:04:15.475526341Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:04:15.475568854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:04:15.481773654Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T18:04:15.622012458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:16.842742515Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=info ts=2026-05-13T18:04:17.157005275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:04:17.160449915Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T18:04:18.480012857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41460"
level=debug ts=2026-05-13T18:04:18.817456124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:04:20.135602087Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T18:04:20.135627364Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-13T18:04:20.555184144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:04:20.622389013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:04:21.180831702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:04:21.185226709Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:04:23.429341033Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:04:23.429366614Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-13T18:04:24.770693118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:04:24.774007654Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:04:25.622558721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:26.720682055Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T18:04:27.135910928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:04:27.139795118Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:04:30.066722773Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-13T18:04:30.066755328Z caller=checkpoint.go:384 msg="flushing series" totalSize="52 MB" series=2
level=warn ts=2026-05-13T18:04:30.621987193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:04:30.776783074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:04:30.784912955Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:04:31.554466942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:04:31.558940684Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:04:33.260449443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:04:33.305616604Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T18:04:35.62240557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:36.59969047Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T18:04:36.599721746Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-13T18:04:38.627366683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:04:38.630422868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:04:39.891276775Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T18:04:40.595250923Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:04:40.598366597Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:04:40.622748111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:04:40.622749668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:40.622776809Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:04:40.622786318Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:04:41.73758111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:04:41.742105427Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:04:43.184161677Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=debug ts=2026-05-13T18:04:43.184186286Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-13T18:04:43.386799594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:04:43.394031879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:04:45.622435565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:46.476011822Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
ts=2026-05-13T18:04:48.482698301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34610"
level=debug ts=2026-05-13T18:04:49.770035041Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T18:04:49.77005962Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T18:04:49.936606133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:04:49.940240963Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T18:04:50.55726531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:04:50.622758535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:50.884579633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:04:53.06217506Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:04:54.121432908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:04:54.124659618Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T18:04:54.124680694Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:04:54.124703585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:04:54.126859507Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T18:04:55.026679844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:04:55.030407162Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:04:55.622303647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:04:56.354945419Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-13T18:04:57.956094475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:04:57.964416114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:04:57.991770941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:04:57.997268685Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:04:59.648121362Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T18:04:59.648151672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-13T18:05:00.622026536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:02.069261639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:05:02.076050856Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:05:02.379889528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:05:02.385278041Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:05:02.683154569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:05:02.687784419Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T18:05:02.939528066Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T18:05:05.622707499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:05.628898419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:05:05.631716819Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:05:06.232689864Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T18:05:07.953168155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:05:07.961864677Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:05:07.972693467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:05:09.528455268Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:05:09.528480806Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=warn ts=2026-05-13T18:05:10.622730169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:10.622731374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:05:10.62276165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:05:10.622769902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:05:12.818555179Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=warn ts=2026-05-13T18:05:15.622094625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:05:16.110385969Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-13T18:05:19.40361338Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-13T18:05:19.418597667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:05:19.425002399Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T18:05:20.558839772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:05:20.622654076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:21.167724269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:05:21.170608028Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:05:22.696163769Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T18:05:23.669383007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:05:23.674022047Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:05:24.157339657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:05:24.159668239Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:05:24.46169959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:05:24.46173269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:05:24.465008364Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T18:05:25.622859311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:05:25.989215999Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T18:05:25.989235471Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=debug ts=2026-05-13T18:05:29.280818822Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T18:05:30.622416137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:31.099132768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:05:31.101824549Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:05:32.573841435Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-13T18:05:35.265669528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:05:35.273216828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:05:35.622701226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:05:35.866589723Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T18:05:37.079767453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:05:37.083325369Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:05:37.222739463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:05:37.225931733Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:05:39.159770874Z caller=checkpoint.go:372 msg="writing series" size="741 kB"
level=debug ts=2026-05-13T18:05:39.159793841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T18:05:40.594567449Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:05:40.598701633Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:05:40.621979969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:40.621983964Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:05:40.622000938Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:05:40.622007804Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:05:42.452116254Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-13T18:05:42.805273624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:05:42.809137314Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T18:05:42.809159575Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:05:42.809181748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:05:42.812521873Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:05:43.832425926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:05:43.834776321Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T18:05:43.969930659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:05:43.974548009Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T18:05:45.622884843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:05:45.74440263Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-13T18:05:48.12017905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:05:48.123747734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:05:49.037080985Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T18:05:50.560454924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:05:50.622128127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:05:50.934715124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:05:50.942111656Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:05:50.942128815Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:05:50.942149881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:05:50.94891948Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:05:52.330408431Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-13T18:05:53.165336593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:05:53.167061797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T18:05:53.846260413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:05:53.850014845Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T18:05:55.024692361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:05:55.030454722Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:05:55.34140366Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:05:55.622743255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:05:55.622842622Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T18:05:57.68338171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:05:57.691040079Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:05:58.915933028Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=info ts=2026-05-13T18:05:59.424008886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:05:59.427526596Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:06:00.442505192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:06:00.444860618Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:06:00.622638039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:06:02.208049791Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-13T18:06:02.726535982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:06:02.731224544Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:06:03.408454595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:06:03.415761687Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:06:05.501075162Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-13T18:06:05.501098805Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T18:06:05.622354073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:06:08.795430687Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T18:06:08.795455736Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=info ts=2026-05-13T18:06:09.949482141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:06:09.952937837Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:06:10.621895889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:06:10.621942509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:06:10.621953046Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:06:10.621977873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:06:11.524937191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:06:11.531005823Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:06:12.085953384Z caller=checkpoint.go:372 msg="writing series" size="426 kB"
level=info ts=2026-05-13T18:06:14.442801597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:06:14.445711315Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:06:15.378921959Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T18:06:15.622768701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:06:16.486279873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:06:16.489261385Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T18:06:18.488273384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47620"
level=debug ts=2026-05-13T18:06:18.671733384Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-13T18:06:20.141866079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:06:20.146465626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T18:06:20.562118786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:06:20.622468928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:06:21.963960925Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-13T18:06:25.256660727Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-13T18:06:25.622174774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:06:25.980528679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:06:25.987920722Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:06:27.366342059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:06:27.370713171Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:06:28.549088315Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-13T18:06:29.645666409Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:06:29.645724156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:06:29.649427646Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:06:30.622192962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:06:30.903833849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:06:30.911143887Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:06:31.842337992Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T18:06:34.677686268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:06:34.680718472Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:06:35.135472059Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=debug ts=2026-05-13T18:06:35.135491947Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-13T18:06:35.621921832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:06:37.594304893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:06:37.599641148Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:06:38.427108351Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T18:06:39.549147693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:06:39.556212528Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:06:40.595510804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:06:40.598609678Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:06:40.621936211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:06:40.621943068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:06:40.621959897Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:06:40.62197603Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:06:40.65728936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:06:40.663276052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:06:41.721152946Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T18:06:41.721178346Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-13T18:06:42.431993568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:06:42.436724803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:06:43.949398326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:06:43.952711975Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:06:44.251614579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:06:44.254098348Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:06:44.476757888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:06:44.480718362Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T18:06:45.013470479Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T18:06:45.621958875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:06:47.277572429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:48784"
level=debug ts=2026-05-13T18:06:48.3054919Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
ts=2026-05-13T18:06:50.563857785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:06:50.62273204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:06:50.942339792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:06:51.450298789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:06:51.453057428Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:06:51.598719337Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-13T18:06:54.890580479Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-13T18:06:55.622786861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:06:58.183834779Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T18:07:00.24634692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:07:00.24900187Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:07:00.622922408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:01.342429395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:07:01.476852396Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-13T18:07:02.458980689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:07:02.459809028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:07:02.462661374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:07:02.462663154Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:07:04.769186951Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-13T18:07:05.621987826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:08.061926315Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-13T18:07:08.321335144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:07:08.32852828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:07:08.328549207Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:07:08.328577841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:07:08.334898103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:07:10.622614853Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:07:10.622647334Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:07:10.622657637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:07:10.62265491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:11.354683164Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-13T18:07:12.239700543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:07:12.242849412Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:07:12.926750395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:07:12.930005998Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:07:14.647617245Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-13T18:07:14.647640815Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
ts=2026-05-13T18:07:15.02958837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:47426"
level=warn ts=2026-05-13T18:07:15.622518038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:07:16.461732653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:07:16.464456282Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T18:07:17.2786134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:59952"
level=debug ts=2026-05-13T18:07:17.939552955Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T18:07:18.063918843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:07:18.066983335Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T18:07:18.492953198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50082"
level=info ts=2026-05-13T18:07:19.47793325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:07:19.480639336Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T18:07:20.564805953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:07:20.622632345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:21.232508782Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-13T18:07:24.525541579Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T18:07:24.959421487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:07:24.963288104Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:07:25.399868492Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:07:25.622463634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:07:26.332667926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:07:26.337445579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:07:26.337465999Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:07:26.337490418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:07:26.341786354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:07:26.627555006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:07:26.633501516Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:07:27.36653002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:07:27.372550418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:07:27.818122866Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T18:07:28.082793155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="905 kB" avg_comp="905 kB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T18:07:28.086479186Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2242ab32%3A19e22657705%3Acdf233de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T18:07:28.086498688Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:07:28.08652244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="611 kB" avg_comp="611 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:07:28.089232516Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:07:28.543368897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:07:28.550531064Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T18:07:30.622257751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:31.110288972Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T18:07:31.745083399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:07:31.748190636Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:07:32.110227163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:07:32.114260504Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:07:32.114277639Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:07:32.114302502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:07:32.117780124Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:07:34.402709074Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-13T18:07:35.622411609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:07:36.15461311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:07:36.160981179Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:07:36.161000446Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:07:36.161021057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:07:36.163294944Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:07:37.696019358Z caller=checkpoint.go:372 msg="writing series" size="616 kB"
level=info ts=2026-05-13T18:07:37.859135144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:07:37.864051512Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:07:40.117855965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:07:40.121226938Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:07:40.595234607Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:07:40.598288612Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:07:40.622689331Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:07:40.622691793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:40.6227086Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:07:40.622723697Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:07:40.988447405Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-13T18:07:43.955410249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:07:43.959107594Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:07:43.959126988Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:07:43.959147116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:07:43.96157788Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:07:44.281961066Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-13T18:07:44.28198275Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
ts=2026-05-13T18:07:45.030787328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:58028"
level=info ts=2026-05-13T18:07:45.39626831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:07:45.398867065Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:07:45.622416926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:07:47.27959666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:51944"
level=debug ts=2026-05-13T18:07:47.574080032Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T18:07:47.90515328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:07:47.90954476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T18:07:48.495326606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57746"
ts=2026-05-13T18:07:50.56635826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:07:50.622276524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:07:50.866946872Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-05-13T18:07:51.381243648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:07:53.520925833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:07:53.524141336Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:07:53.654670118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:07:53.659087813Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:07:53.797610337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:07:53.800255633Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:07:54.159235066Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-13T18:07:55.621952998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:07:56.543406426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:07:56.549796277Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:07:57.45237612Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T18:07:58.944722823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:08:00.088989236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:08:00.09139081Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:08:00.46465541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:08:00.467292259Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:08:00.621911996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:08:00.744424773Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=info ts=2026-05-13T18:08:02.103597092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:08:02.108767697Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:08:04.037531913Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T18:08:04.140929697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:08:04.143972124Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:08:04.813931067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:08:04.817398991Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T18:08:05.622282637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:06.370862352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:08:06.373264491Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:08:06.373283965Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:08:06.373303652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:08:06.375297583Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:08:07.346173187Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-13T18:08:07.346207404Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=info ts=2026-05-13T18:08:10.622023518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:08:10.622052585Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:08:10.62206002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:08:10.622071025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:10.646127095Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-13T18:08:10.647984289Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.025886449s
level=info ts=2026-05-13T18:08:10.855007871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:08:10.861035303Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:08:11.413919928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:08:11.417150193Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:08:11.846308713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:08:11.852698327Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:08:13.424821233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:08:13.427643854Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-13T18:08:15.03213091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:49598"
level=warn ts=2026-05-13T18:08:15.622333764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:16.433515239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:08:16.43765357Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T18:08:18.49727899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43540"
level=info ts=2026-05-13T18:08:18.513692915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:08:18.517167672Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:08:19.192299538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:08:19.198896303Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T18:08:20.568445551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:08:20.622091301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:25.438425044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:08:25.441816552Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T18:08:25.622411884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:29.947655782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:08:29.950644153Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:08:29.950670831Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:08:29.950701322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:08:29.953006135Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:08:30.181683624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:08:30.184441114Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:08:30.622459385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:08:31.826032079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:08:35.622339912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:08:40.596577577Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:08:40.598701992Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:08:40.621981869Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T18:08:40.621993562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:08:40.62201383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:08:40.622023609Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:08:40.622030497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:40.62211772Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-13T18:08:40.622127811Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T18:08:40.622145734Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.017µs success=true
level=debug ts=2026-05-13T18:08:40.622462803Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=info ts=2026-05-13T18:08:43.020209322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:08:43.025546706Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:08:43.025567788Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:08:43.025590416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:08:43.030106422Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:08:43.34893536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:08:43.35144367Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:08:43.496922307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:08:43.499710266Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:08:43.915985839Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=warn ts=2026-05-13T18:08:45.622429882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:08:46.109789286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:08:47.20837694Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T18:08:47.282517992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:57388"
level=info ts=2026-05-13T18:08:49.346287119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:08:49.349279896Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:08:49.349297973Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:08:49.34931692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:08:49.351193071Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:08:50.009755338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:08:50.012241958Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:08:50.5008123Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
ts=2026-05-13T18:08:50.570315238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:08:50.622414355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:08:51.435985833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:08:51.439818553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:08:53.414451267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:08:53.417861347Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:08:53.794083165Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-13T18:08:54.703261486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:08:54.710462657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:08:54.780560458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:08:54.787436622Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T18:08:55.6227329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:08:56.042032047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:08:56.402634403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:08:56.40837488Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:08:57.086831709Z caller=checkpoint.go:372 msg="writing series" size="546 kB"
level=debug ts=2026-05-13T18:08:57.086854463Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-13T18:08:58.811070447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:08:58.817980648Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:09:00.379036583Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T18:09:00.622901522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:09:01.313641312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:09:01.318462711Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:09:01.88682939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:09:01.889314382Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:09:03.672174856Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T18:09:05.194507286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:09:05.197237932Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T18:09:05.622947729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:09:06.585603068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:09:06.588904123Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:09:06.96485916Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T18:09:07.912646043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:09:07.916147559Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T18:09:08.898633132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:09:08.905609615Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:09:10.257810378Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T18:09:10.257837519Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T18:09:10.622024369Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:09:10.62205113Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T18:09:10.622041979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:09:10.62205867Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:09:13.550126791Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T18:09:15.622209747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:16.84266777Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
ts=2026-05-13T18:09:18.501268139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38344"
level=info ts=2026-05-13T18:09:20.018162793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:09:20.02637489Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:09:20.135263552Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T18:09:20.135291755Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-13T18:09:20.342527695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:09:20.346185774Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T18:09:20.57231822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:09:20.622095291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:21.904041806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:09:22.474620431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:09:22.478551079Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:09:23.429513087Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:09:23.42953409Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-13T18:09:23.725520194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:09:23.72843126Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:09:25.622165442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:26.719955863Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T18:09:29.522985652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:09:29.526277276Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:09:30.057906852Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-13T18:09:30.057930059Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=2
level=warn ts=2026-05-13T18:09:30.622430408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:33.305419661Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T18:09:35.621955407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:36.599495878Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T18:09:36.599518316Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-13T18:09:36.922518311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:09:36.928475008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:09:39.891390469Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=info ts=2026-05-13T18:09:40.350436823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:09:40.353524939Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:09:40.59523492Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:09:40.59836519Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:09:40.62273707Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:09:40.622769328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:09:40.622776968Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:09:40.622770288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:09:41.434804071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:09:41.443033923Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:09:42.86792714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:09:42.870993453Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:09:43.184198889Z caller=checkpoint.go:372 msg="writing series" size="493 kB"
level=debug ts=2026-05-13T18:09:43.18421928Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-05-13T18:09:45.037367713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:41012"
level=warn ts=2026-05-13T18:09:45.622776312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:46.476452366Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-13T18:09:46.702027048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:09:48.057561834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:09:49.070307397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:09:49.07397152Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:09:49.770568178Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T18:09:49.77059026Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
ts=2026-05-13T18:09:50.573526838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:09:50.622398508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:09:51.248628114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:09:51.253912196Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:09:52.088352502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:09:52.091423001Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:09:52.833431625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:09:52.837981823Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:09:53.061756007Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:09:54.140933435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:09:54.148038689Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:09:55.414263105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:09:55.41734349Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T18:09:55.62207147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:09:56.35416663Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-13T18:09:59.648625703Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T18:09:59.648657587Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-13T18:10:00.622625531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:10:01.577666779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:10:01.585360392Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:10:02.231299141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:10:02.233646451Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:10:02.939683663Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T18:10:03.48019193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:10:03.485424805Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:10:04.507638665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:10:04.510100389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:10:05.621933968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:10:06.232347306Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T18:10:07.253680684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:10:07.25689882Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:10:09.529542111Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-13T18:10:09.529564932Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=warn ts=2026-05-13T18:10:10.622589883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:10:10.62260506Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:10:10.62264155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:10:10.622651762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:10:12.819017792Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=warn ts=2026-05-13T18:10:15.622358361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:10:16.110172067Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
ts=2026-05-13T18:10:17.293493411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:52328"
level=info ts=2026-05-13T18:10:19.271694115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:10:19.277220632Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:10:19.315530983Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:10:19.403067218Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-13T18:10:19.485143756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:10:19.48795867Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T18:10:20.57508106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:10:20.621962821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:10:22.695630436Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T18:10:23.531145759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:10:23.536265001Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:10:23.605173633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:10:23.606699545Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T18:10:25.622368121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:10:25.98946247Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T18:10:25.989482233Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=5
level=debug ts=2026-05-13T18:10:29.281693879Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T18:10:30.622797115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:10:31.384645716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:10:31.387433222Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:10:32.57407083Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-13T18:10:35.621988817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:10:35.867229122Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T18:10:36.372398278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:10:36.379452446Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:10:36.379470112Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:10:36.379505329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:10:36.385587983Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:10:37.837814457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:10:37.840767296Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:10:39.159869663Z caller=checkpoint.go:372 msg="writing series" size="816 kB"
level=debug ts=2026-05-13T18:10:39.159916685Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T18:10:39.496058539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:10:39.499116486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:10:39.499134183Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:10:39.499153111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:10:39.501776992Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:10:39.833928388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:10:39.837058401Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:10:40.106755812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:10:40.111103813Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:10:40.595507244Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:10:40.598629848Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:10:40.621942205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:10:40.621960454Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T18:10:40.621959372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:10:40.621970056Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:10:41.703325605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:10:41.710212708Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:10:42.027126101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:10:42.030393351Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:10:42.452564869Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-13T18:10:45.616222287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:10:45.618613732Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T18:10:45.622665716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:10:45.673763767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:10:45.676146047Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:10:45.744428697Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
ts=2026-05-13T18:10:48.507351677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54370"
level=info ts=2026-05-13T18:10:48.91208547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:10:48.918360341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:10:49.037979572Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T18:10:50.576220536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:10:50.622583837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:10:51.128813708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:10:51.137233296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:10:52.330381727Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-13T18:10:53.590517686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:10:55.061790652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:10:55.064724163Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:10:55.621998544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:10:55.623063406Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T18:10:57.092227928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:10:57.095688939Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:10:58.624122938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:10:58.626676697Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T18:10:58.915671924Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
level=warn ts=2026-05-13T18:11:00.622078942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:01.850569302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:11:01.857383444Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:11:02.208726986Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-13T18:11:02.852641828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:11:02.856032518Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:11:05.501484394Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-13T18:11:05.501508311Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-13T18:11:05.622702488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:05.779299271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:11:05.781682479Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:11:08.794364014Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T18:11:08.7943856Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=debug ts=2026-05-13T18:11:10.261720459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T18:11:10.622618225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:10.622614485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:11:10.62264695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:11:10.622655349Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:11:12.087143119Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=info ts=2026-05-13T18:11:12.783733507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:11:12.786234107Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:11:13.757560521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:11:13.762718967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:11:15.379059513Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T18:11:15.622241954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:11:18.671772698Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
ts=2026-05-13T18:11:20.578048177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:11:20.622389139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:11:21.964265254Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-13T18:11:22.888630068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:11:22.895093861Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:11:24.310928332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:11:24.317316787Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:11:24.612064018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:11:24.616405603Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:11:25.256638911Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=warn ts=2026-05-13T18:11:25.62268013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:11:26.742903739Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:11:27.215306203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:11:27.219242654Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:11:28.550067374Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-13T18:11:28.550089671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-13T18:11:28.882984804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:11:28.886691674Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:11:30.622538001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:31.466444132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:11:31.468947949Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:11:31.842024065Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T18:11:33.478402361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:11:33.482077746Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:11:35.135382957Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-13T18:11:35.622548303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:38.368570925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:11:38.373990452Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:11:38.427638303Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T18:11:39.494580969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:11:39.498310856Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:11:40.595374903Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:11:40.598483059Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:11:40.621828458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:11:40.621856237Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:11:40.621866113Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:11:40.621912356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:41.13252099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:11:41.140429915Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:11:41.723387535Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T18:11:41.723421905Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
level=debug ts=2026-05-13T18:11:45.013412767Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T18:11:45.622302177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:11:48.305734165Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-13T18:11:49.255830262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T18:11:50.579096977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:11:50.622703944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:51.001390699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:11:51.003975414Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:11:51.374590914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:11:51.378686131Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:11:51.598743159Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-13T18:11:51.921772406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:11:52.376935035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:11:52.382231635Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:11:52.382265723Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:11:52.382297717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:11:52.385637332Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:11:54.891851961Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-13T18:11:55.622462885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:11:56.770475854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:11:56.775302858Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:11:58.012406326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T18:11:58.015076125Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e2242ab0d%3A19e2242ab33%3A5223194c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T18:11:58.015097209Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:11:58.015120597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:11:58.017748535Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:11:58.184253952Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T18:12:00.622084713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:12:01.477018756Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T18:12:04.769801547Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-13T18:12:05.62240948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:12:06.070298309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:12:06.075505492Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:12:08.062338614Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-13T18:12:08.26612782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:12:08.269512823Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:12:08.443391656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:12:08.447464213Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:12:10.087682146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:12:10.095089905Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:12:10.622288555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:12:10.622331549Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:12:10.622341053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:12:10.622338556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:12:11.323255018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:12:11.329371102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:12:11.354440149Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-13T18:12:12.199836398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:12:12.202505103Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:12:14.63339424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:12:14.64105081Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:12:14.641077651Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:12:14.641104869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:12:14.647030685Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=debug ts=2026-05-13T18:12:14.647049915Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=error ts=2026-05-13T18:12:14.648433844Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T18:12:15.622098833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:12:15.735417035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:12:15.73919169Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:12:16.316639977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:12:16.32012437Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:12:17.589209894Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:12:17.785861745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:12:17.789202522Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:12:17.939446396Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-13T18:12:18.512895096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34692"
ts=2026-05-13T18:12:20.579951335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:12:20.622468985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:12:21.232958712Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-13T18:12:24.132528279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:12:24.13595166Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:12:24.525007447Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T18:12:25.622913737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:12:25.839630677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:12:25.846072341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:12:27.817954495Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T18:12:27.898349972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:12:27.900942506Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:12:28.111509034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:12:28.115145782Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T18:12:28.115164822Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:12:28.115186266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:12:28.117853608Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:12:28.25825087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:12:28.264435344Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T18:12:30.622060413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:12:31.11099942Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-13T18:12:34.403780432Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-05-13T18:12:35.622914327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:12:36.84981392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:12:36.852526365Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:12:37.696317974Z caller=checkpoint.go:372 msg="writing series" size="668 kB"
level=info ts=2026-05-13T18:12:39.539427611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:12:39.543111754Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:12:40.595858182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:12:40.598956045Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:12:40.622341854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:12:40.622341029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:12:40.622363527Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:12:40.622373849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:12:40.988257132Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-13T18:12:40.988278371Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-13T18:12:42.364385127Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:12:44.040516531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:12:44.044210357Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:12:44.28120725Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=info ts=2026-05-13T18:12:45.387058876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:12:45.389353315Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:12:45.622693898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:12:47.574178933Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-13T18:12:48.578228926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T18:12:50.580930419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:12:50.622532149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:12:50.867019241Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-13T18:12:51.699179817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:12:51.702544129Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:12:54.160024014Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T18:12:54.305499416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:12:54.312164215Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T18:12:55.622257059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:12:57.131009395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:12:57.137217976Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:12:57.452301137Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T18:12:57.452324499Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T18:12:58.883577595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:12:58.889849827Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:13:00.375771972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:13:00.382322618Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T18:13:00.621972607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:13:00.745397862Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
level=info ts=2026-05-13T18:13:03.564469541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:13:03.568649624Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:13:04.037166077Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T18:13:04.585651654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:13:04.588231607Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:13:04.588250559Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:13:04.588271994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:13:04.5910373Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:13:04.860675359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:13:04.865214822Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:13:05.208300939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:13:05.21083517Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:13:05.622564421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:13:07.348289158Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-13T18:13:07.348319676Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=debug ts=2026-05-13T18:13:07.547705254Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:13:09.334484813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:13:09.339375891Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:13:09.917066544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:13:09.92090896Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:13:10.622597756Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:13:10.622635851Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:13:10.62264482Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:13:10.622758538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:13:10.628317557Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-13T18:13:10.629489892Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007190075s
level=info ts=2026-05-13T18:13:11.406444625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:13:11.409269754Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:13:11.453367774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:13:11.460768474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:13:12.876560209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:13:12.880043346Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:13:15.181175145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:13:15.184417955Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:13:15.622560569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:13:17.303673211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:34814"
ts=2026-05-13T18:13:20.582407046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:13:20.622636811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T18:13:25.622152491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:13:28.003117758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:13:28.013763848Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:13:30.62193125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T18:13:35.622746395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:13:35.825236394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:13:35.828594138Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:13:38.026613333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:13:38.030327702Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:13:39.272925485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:13:39.277388173Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:13:40.594764108Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:13:40.598907412Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:13:40.622239342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:13:40.62226579Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:13:40.622263732Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T18:13:40.622286557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:13:40.622309368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:13:40.622452491Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-13T18:13:40.622462589Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T18:13:40.622480353Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.221µs success=true
level=debug ts=2026-05-13T18:13:40.622765697Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-13T18:13:41.202325315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:13:41.204895685Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:13:41.442308544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:13:41.97195745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:13:43.876085362Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-13T18:13:45.305678311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:13:45.308212597Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:13:45.622356585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:13:46.728694961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:13:46.735849428Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:13:47.128999804Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T18:13:47.240378934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:13:47.242722493Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:13:49.993343839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:13:49.998250795Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:13:49.998270322Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:13:49.998292918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:13:50.002554966Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:13:50.382691428Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=info ts=2026-05-13T18:13:50.518011064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:13:50.523571123Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:13:50.523589388Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:13:50.523610323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:13:50.528635416Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T18:13:50.583293643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:13:50.621928204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:13:52.608916805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:13:52.611479456Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:13:52.61149821Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:13:52.611520092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:13:52.613484627Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:13:53.635637243Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-13T18:13:55.332521837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:13:55.340098319Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:13:55.562654511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:13:55.565795888Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:13:55.622030967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:13:56.888371497Z caller=checkpoint.go:372 msg="writing series" size="576 kB"
level=debug ts=2026-05-13T18:13:56.888395094Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-13T18:13:57.726628208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:13:57.73502731Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:13:57.735049682Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:13:57.735078747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:13:57.741208649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:14:00.141713563Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-05-13T18:14:00.354228421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:14:00.358890899Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T18:14:00.622353058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:14:00.74263985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:14:00.745117976Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:14:00.773682692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:14:00.777076886Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:14:00.777096066Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:14:00.777118561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:14:00.780163591Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:14:03.394861247Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T18:14:03.43830392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:14:03.444544003Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T18:14:05.622112983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:14:06.64691466Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-13T18:14:09.493440582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:14:09.500126751Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:14:09.901558908Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T18:14:09.901598552Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T18:14:10.622369782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:14:10.622408444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:14:10.622416509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:14:10.622409088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:14:11.165796768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:14:11.172461695Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:14:11.671151664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:14:11.673763271Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:14:13.153264705Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T18:14:14.183540759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:14:14.187549619Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T18:14:15.622118128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:14:16.211522231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:14:16.406431755Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
ts=2026-05-13T18:14:17.307268825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:58558"
ts=2026-05-13T18:14:18.521661481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44366"
level=info ts=2026-05-13T18:14:19.315608266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:14:19.318300926Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:14:19.659626683Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T18:14:19.659651001Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-13T18:14:20.584846489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:14:20.621944009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:14:21.68453836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:14:21.688407178Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:14:22.915179068Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:14:22.915200721Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-13T18:14:23.193292716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:14:23.196823595Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:14:24.147296757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:14:24.150603481Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T18:14:25.622150654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:14:26.165327501Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-13T18:14:29.464116492Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-13T18:14:29.464142153Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=2
level=warn ts=2026-05-13T18:14:30.622558092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:14:32.67127784Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-13T18:14:33.903308037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:14:33.904802929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T18:14:34.272818766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:14:34.279174929Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:14:35.622501124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:14:35.870434785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:14:35.875381694Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:14:35.925304423Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T18:14:35.925325277Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T18:14:38.565929775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:14:38.570271912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:14:38.747004087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:14:38.752421368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:14:39.177446719Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=info ts=2026-05-13T18:14:39.223614599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:14:39.226013575Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:14:39.22603205Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:14:39.22605217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:14:39.22797707Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:14:40.558653721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:14:40.567112074Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:14:40.594936834Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:14:40.599050733Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:14:40.622396314Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:14:40.622419327Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:14:40.62243101Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:14:40.622447783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:14:40.665616855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:14:40.668050006Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:14:40.668066173Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:14:40.668084528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:14:40.670099681Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:14:42.431183082Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=debug ts=2026-05-13T18:14:42.431205302Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-05-13T18:14:45.05416962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:44306"
level=warn ts=2026-05-13T18:14:45.622164592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:14:45.683703869Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-13T18:14:46.521587312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:14:46.524313351Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:14:46.52433452Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:14:46.524355424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:14:46.527124676Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:14:47.779816477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:14:47.784538712Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:14:48.937423856Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T18:14:48.937451718Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T18:14:49.207806877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:14:49.211847747Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:14:49.570817062Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:14:50.576855832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:14:50.580863501Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T18:14:50.586621612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:14:50.622637731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:14:50.703922748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:14:50.706517264Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:14:52.189274094Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:14:52.710919612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:14:52.718270575Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:14:53.431450172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:14:53.43408712Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:14:54.08285669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:14:54.086142162Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:14:55.442785544Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-13T18:14:55.622510098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:14:58.697306203Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T18:14:58.697332875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-13T18:15:00.622539743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:01.948234542Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T18:15:02.933328725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:15:02.937362802Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:15:04.088631883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:15:04.09186045Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:15:05.201854552Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T18:15:05.622558635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:08.458838401Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:15:08.458867214Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=info ts=2026-05-13T18:15:09.823090871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:15:09.826958546Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:15:10.622368313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:15:10.622411407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:15:10.62242301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:15:10.622473133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:15:11.485988082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:15:11.49030418Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:15:11.708028358Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T18:15:14.960213258Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-13T18:15:15.622700675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:15:16.17869578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:15:16.181793716Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:15:16.181811869Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:15:16.181830337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:15:16.183901166Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T18:15:17.31024382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:52114"
level=debug ts=2026-05-13T18:15:18.214000306Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-13T18:15:19.693196754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:15:19.69822939Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T18:15:20.587774014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:15:20.622607107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:15:20.747932244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:15:20.750869954Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:15:21.466486957Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T18:15:23.533972928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:15:23.819008749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:15:23.821377903Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:15:24.582904179Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:15:24.657655442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:15:24.662479523Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:15:24.662499393Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:15:24.662522711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:15:24.666807796Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:15:24.721471028Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T18:15:24.721490879Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=info ts=2026-05-13T18:15:25.079964615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:15:25.087019336Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T18:15:25.62289361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:15:26.236666635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:15:26.244937131Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:15:26.654167443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:15:26.657286713Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:15:27.024328097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:15:27.026576037Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:15:27.972236615Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T18:15:28.2903194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:15:28.292662119Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:15:30.622318693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:31.225601945Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-13T18:15:32.004944187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:15:32.008313131Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:15:34.478293553Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-13T18:15:35.621995965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:37.732377872Z caller=checkpoint.go:372 msg="writing series" size="709 kB"
level=debug ts=2026-05-13T18:15:37.732402202Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T18:15:39.141621465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:15:39.148096099Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:15:40.596223741Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:15:40.598267846Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T18:15:40.599552546Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T18:15:40.599600436Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976328 n=1
level=debug ts=2026-05-13T18:15:40.599641804Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976328 n=1
level=info ts=2026-05-13T18:15:40.62263389Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:15:40.622647253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:40.622659061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:15:40.622669142Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:15:40.984760092Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-13T18:15:42.786602955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:15:42.793750596Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:15:43.582925916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:15:43.588050223Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:15:44.237893547Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-13T18:15:44.261914212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:15:44.265614955Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T18:15:45.622131662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:15:46.665786504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:15:46.669473707Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T18:15:47.311715095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:43866"
level=debug ts=2026-05-13T18:15:47.490839685Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T18:15:48.528110869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38558"
level=info ts=2026-05-13T18:15:48.774139776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:15:48.776954401Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T18:15:50.58853923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:15:50.622447457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:50.743371986Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-13T18:15:53.996573193Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T18:15:55.622400574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:15:57.249796641Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-13T18:15:57.293035683Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:15:59.434005207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:15:59.441821808Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:16:00.503070501Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T18:16:00.622356845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:16:00.633511051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:16:00.636383969Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:16:00.908527429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:16:00.910845004Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:16:01.961953765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:16:02.021362491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:16:02.023817342Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:16:03.636449863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:16:03.642136036Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:16:03.755561625Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T18:16:04.161929385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:16:04.164799501Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:16:05.622114169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:16:05.629253883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:16:05.632550242Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:16:07.01291484Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T18:16:07.012944843Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=9
level=info ts=2026-05-13T18:16:09.224608646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:16:09.232838032Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:16:10.262345025Z caller=checkpoint.go:372 msg="writing series" size="598 kB"
level=info ts=2026-05-13T18:16:10.622849693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:16:10.622917168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:16:10.622932583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:16:10.62292131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:13.514594013Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T18:16:14.515523574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:16:14.519152034Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:16:15.622539374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:16.768242543Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-13T18:16:18.295512099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:16:18.303052156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:16:20.021000932Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-13T18:16:20.035975365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:16:20.039653367Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T18:16:20.589896184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:16:20.622830567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:23.273958014Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-13T18:16:23.273981081Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-13T18:16:24.386190301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:16:25.622326694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:26.526765138Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-13T18:16:26.805733245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:16:26.809717103Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:16:29.551248999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:16:29.554966571Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:16:29.779624186Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-13T18:16:30.622447179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:33.033145102Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-13T18:16:34.249301201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:16:34.252334212Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:16:34.536008702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:16:34.543174367Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:16:34.939175203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:16:34.943767124Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T18:16:35.622111169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:16:35.700377195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:16:35.703950229Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:16:36.285983832Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-13T18:16:39.541386981Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T18:16:39.54141099Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=debug ts=2026-05-13T18:16:40.595529091Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:16:40.598632058Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:16:40.621929319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:16:40.621948532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:16:40.621956059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:16:40.621972675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:42.791406383Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-13T18:16:43.333400164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:16:43.339912966Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T18:16:45.061091949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:43756"
level=warn ts=2026-05-13T18:16:45.622848802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:46.045751249Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
ts=2026-05-13T18:16:47.314928824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:38368"
level=info ts=2026-05-13T18:16:48.870329471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:16:48.875192002Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:16:48.990890243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:16:49.297718243Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-13T18:16:49.668789911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:16:49.671838266Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T18:16:50.591991391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:16:50.622069814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:16:51.201284541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:16:51.203693527Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:16:52.379587754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:16:52.55120138Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-13T18:16:52.706831148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:16:52.713178142Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:16:53.975908038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:16:53.978801395Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:16:55.622588917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:16:55.803806425Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T18:16:58.347896019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:16:58.350785389Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:16:58.362834659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:16:58.366968053Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:16:59.057188491Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-13T18:17:00.622276355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:02.310357461Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T18:17:05.389224525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:17:05.395586039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:17:05.56297377Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=debug ts=2026-05-13T18:17:05.562996493Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T18:17:05.622945315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:17:08.36380145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:17:08.367167153Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:17:08.815918635Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-13T18:17:10.621836788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:17:10.621895493Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:17:10.621906385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:17:10.621957547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:12.069337634Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-13T18:17:13.206400734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:17:13.210539621Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:17:13.21055752Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:17:13.210579244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:17:13.213628025Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:17:13.226770291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:17:13.229085271Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T18:17:15.062201981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:58948"
level=debug ts=2026-05-13T18:17:15.322346802Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T18:17:15.52803973Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:17:15.622683342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:17:18.168643099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:17:18.171671977Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:17:18.574750502Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-13T18:17:19.927417764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:17:19.931629468Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T18:17:20.593470318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:17:20.622542427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:21.828270742Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T18:17:22.040110362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:17:22.042679723Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:17:23.838919543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:17:23.843526877Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:17:25.080723305Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T18:17:25.622548136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:28.333677304Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T18:17:29.42745514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:17:29.431113888Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T18:17:30.621931166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:31.587176647Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-13T18:17:32.903763604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:17:32.9069917Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:17:34.457427211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:17:34.464135169Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:17:34.464153627Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:17:34.464185999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:17:34.469771269Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:17:34.64616766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:17:34.649053601Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:17:34.841004703Z caller=checkpoint.go:372 msg="writing series" size="671 kB"
level=debug ts=2026-05-13T18:17:34.841029753Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-13T18:17:35.621929452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:38.093129146Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-13T18:17:38.245925038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:17:38.248489258Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:17:40.595225555Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:17:40.598275708Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:17:40.622650939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:17:40.622665027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:17:40.622693055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:17:40.622700992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:17:40.79320066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:17:40.799631756Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:17:41.34622438Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-13T18:17:42.492266568Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:17:42.492313847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="79 kB" avg_comp="79 kB" total_uncomp="401 kB" avg_uncomp="401 kB" idle=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:17:42.494838641Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=debug ts=2026-05-13T18:17:44.599678012Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T18:17:45.62254843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:17:46.758642706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:17:46.760967668Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T18:17:47.319130669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:51540"
level=debug ts=2026-05-13T18:17:47.852666172Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-13T18:17:47.852687718Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T18:17:48.165142889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:17:48.169704022Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:17:48.556807366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:17:48.559864746Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:17:49.933142056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:17:49.936679796Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-13T18:17:50.594503215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:17:50.622444352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:17:50.77527502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:17:50.781277057Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:17:51.105196011Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-13T18:17:52.858178184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:17:53.118945169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:17:53.125238322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:17:54.358286532Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-13T18:17:55.082601942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:17:55.08529864Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T18:17:55.622045479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:17:57.610937479Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-05-13T18:17:58.238745762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:17:58.241312801Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:18:00.622640661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:18:00.8646727Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T18:18:01.196211409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:18:01.198931373Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=info ts=2026-05-13T18:18:03.599480653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:18:03.603086964Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:18:03.73828705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:18:03.74254939Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:18:04.13414245Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-13T18:18:04.134174309Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=5
level=warn ts=2026-05-13T18:18:05.62238136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:05.970692822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:18:05.973192807Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:18:07.369980139Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T18:18:09.375222211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:18:09.38023929Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:18:10.622780812Z caller=checkpoint.go:384 msg="flushing series" totalSize="64 kB" series=1
level=info ts=2026-05-13T18:18:10.622818965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:18:10.622848779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:18:10.622863067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:18:10.622866405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:10.628342755Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-13T18:18:10.629606046Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006987046s
level=info ts=2026-05-13T18:18:10.766769064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:18:10.770043273Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:18:15.622843069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:18:16.484779022Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:18:18.288172662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:18:18.296800109Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T18:18:18.538501776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44148"
level=info ts=2026-05-13T18:18:19.87565517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:18:19.879608678Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:18:19.977845978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:18:19.982025873Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T18:18:20.59648614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:18:20.62261844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:24.469775143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:18:24.472774367Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=warn ts=2026-05-13T18:18:25.622751213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:28.619780219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:18:28.622958834Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:18:30.622172769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:33.835988575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:18:33.840003321Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:18:33.840034759Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:18:33.840058452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:18:33.842797546Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:18:34.680010753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:18:34.683525939Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:18:34.683550026Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:18:34.683584449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:18:34.686781656Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-13T18:18:35.622455384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:36.057807002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:18:36.060906187Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:18:36.697292372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:18:36.69991039Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:18:39.598318308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:18:39.60595724Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:18:39.605982801Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:18:39.60601059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:18:39.615509462Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:18:40.505023107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:18:40.507727144Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:18:40.594864653Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:18:40.598973529Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:18:40.622384845Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T18:18:40.622402168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:40.622397138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:18:40.622425696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:18:40.62243401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:18:40.622710675Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-13T18:18:40.62272007Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T18:18:40.622741644Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.68µs success=true
level=debug ts=2026-05-13T18:18:40.623058195Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-13T18:18:42.135716222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:18:42.137464924Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T18:18:43.08104926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:18:43.088254158Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:18:43.87678093Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-13T18:18:44.957329987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:18:44.965501177Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:18:45.622517284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:45.961248513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:18:45.965439401Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T18:18:45.965474697Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:18:45.965511441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:18:45.969726594Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:18:46.336201591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:18:46.340260516Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:18:47.129962367Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T18:18:47.32272639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:49096"
level=debug ts=2026-05-13T18:18:50.382451509Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
ts=2026-05-13T18:18:50.598475813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:18:50.622707678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:18:50.840757251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:18:50.847308482Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:18:51.596424563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:18:51.596403153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="35 kB" avg_uncomp="35 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:18:51.599852346Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:18:51.89674605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="52 kB" avg_uncomp="52 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:18:51.900205758Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-13T18:18:53.635203667Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-13T18:18:55.622836629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:18:56.891655042Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
level=debug ts=2026-05-13T18:18:56.89169432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-13T18:18:58.328840313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:18:58.332080497Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:18:58.377015915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:18:58.383044282Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:18:58.400314672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:18:58.409176231Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:19:00.141609322Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T18:19:00.622584057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:01.41306672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:19:01.416306229Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:19:03.237857602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:19:03.242363169Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:19:03.396285203Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T18:19:04.115408291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:19:04.118331426Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-13T18:19:05.622375579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:05.842354798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:19:05.846076587Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:19:05.958315192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:19:05.961760147Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:19:06.512698446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:19:06.517924172Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:19:06.549853249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:19:06.554727895Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:19:06.647442662Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=debug ts=2026-05-13T18:19:09.901252329Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T18:19:09.901278157Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T18:19:10.581888062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:19:10.585862825Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:19:10.622507678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:19:10.622543376Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:19:10.622553378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:19:10.622558571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:10.935216727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:19:10.93864353Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:19:12.124726675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:19:12.132442117Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:19:13.153171185Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T18:19:15.621938502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:19:16.280629801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:19:16.280687046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="358 kB" avg_comp="358 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:19:16.288792636Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:19:16.406704536Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=info ts=2026-05-13T18:19:16.642184648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:19:16.645088759Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:19:16.645121692Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:19:16.645152217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:19:16.647854596Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:19:19.660138634Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T18:19:19.660161647Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-13T18:19:20.600264848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:19:20.622445608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:22.85053376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:19:22.855429371Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=debug ts=2026-05-13T18:19:22.914314814Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:19:22.914336654Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-13T18:19:23.672851294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:19:23.675706773Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:19:25.62245539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:19:26.165407206Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T18:19:27.183547399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:19:27.187092725Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:19:27.608662382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:19:27.611368581Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:19:27.978228686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:19:27.98618517Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:19:28.637230589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:19:28.640333071Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:19:29.46799309Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-13T18:19:29.468037158Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=2
level=info ts=2026-05-13T18:19:30.112159673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:19:30.114797172Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-13T18:19:30.622645922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:31.300638049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:19:31.308035674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:19:31.373032915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:19:32.671853447Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-13T18:19:32.996004261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:19:32.999488414Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:19:33.538932941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:19:33.544582847Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:19:35.523849819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:19:35.526660826Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:19:35.622247523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:19:35.925976973Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T18:19:35.92600792Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-13T18:19:39.178142585Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=info ts=2026-05-13T18:19:39.907149594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:19:39.909925873Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T18:19:40.594792326Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:19:40.598917373Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:19:40.622291758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:19:40.62231353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:19:40.622321227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:19:40.622288636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:40.705187885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="384 kB" avg_comp="384 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T18:19:40.705240094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T18:19:40.708610798Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T18:19:42.431215891Z caller=checkpoint.go:372 msg="writing series" size="507 kB"
level=debug ts=2026-05-13T18:19:42.431239736Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-05-13T18:19:45.072219525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:35208"
level=warn ts=2026-05-13T18:19:45.622597128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:19:45.683913398Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-13T18:19:47.742959412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:19:47.745987062Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:19:48.937763598Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T18:19:48.937791014Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T18:19:49.465262152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:19:49.468548155Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T18:19:50.601821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:19:50.622709963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:51.271824148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:19:51.275528023Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:19:52.189806322Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:19:52.317853287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:19:52.324062747Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:19:53.531200897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:19:53.540255722Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:19:55.315170301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:19:55.318502763Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:19:55.443025249Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-13T18:19:55.622466361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:19:56.8918866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:19:56.895017062Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T18:19:58.03392928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:19:58.03861264Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:19:58.697116466Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T18:19:58.697138044Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-05-13T18:19:59.969212725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:19:59.971843549Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:20:00.622614625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:00.756919406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:20:00.759981423Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:20:01.949119434Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T18:20:04.200138529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:20:04.204000173Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:20:05.201819241Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-13T18:20:05.578790305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:20:05.622589818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:08.449070347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:20:08.4524514Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:20:08.458120003Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:20:08.458181682Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T18:20:08.749607651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:20:08.753042158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:20:09.514145952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:20:09.517293649Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=info ts=2026-05-13T18:20:10.622591272Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:20:10.622627996Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:20:10.622636867Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:20:10.622651323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:20:11.708606963Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=info ts=2026-05-13T18:20:12.937616057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:20:12.940750975Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-13T18:20:14.961078815Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T18:20:15.53768889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:20:15.541171583Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T18:20:15.622323763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:20:18.21421434Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
ts=2026-05-13T18:20:18.546822308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:39432"
level=info ts=2026-05-13T18:20:19.514861214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:20:19.518786678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:20:19.518816894Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:19.518849145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:20:19.521273803Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:20:19.816312172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:20:19.824733818Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:20:19.824765746Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:19.824799154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:20:19.83109419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T18:20:20.603098791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:20:20.622159905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:21.121777215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:20:21.124288761Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:20:21.467285653Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T18:20:22.198203659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:20:22.200656278Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T18:20:23.439497169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:20:24.090292926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:20:24.098911082Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:20:24.721259822Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T18:20:24.721284505Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=info ts=2026-05-13T18:20:25.580559983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="397 kB" avg_comp="397 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:20:25.583935064Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-13T18:20:25.622165121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:26.960021722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:20:26.966063918Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:20:26.96609464Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:26.966127295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:20:26.972181848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:20:27.972954813Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=warn ts=2026-05-13T18:20:30.62193113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:20:31.226434974Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-13T18:20:32.035923817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:20:33.447082168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:20:33.449636573Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:20:34.478753811Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T18:20:34.741867115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:20:34.744609742Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:20:35.622494036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:35.650602582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:20:35.658051118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:20:37.732392977Z caller=checkpoint.go:372 msg="writing series" size="791 kB"
level=debug ts=2026-05-13T18:20:37.732412338Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T18:20:38.345752174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:20:38.349319668Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:20:39.828621557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:20:39.833994019Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:20:40.464617209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:20:40.468377901Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:20:40.594867542Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:20:40.598962775Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:20:40.622351976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:40.622354765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:20:40.622376287Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:20:40.622384892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:20:40.984439391Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T18:20:42.918649781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:20:42.921241003Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:20:42.921260004Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:42.921283104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:20:42.923316825Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:20:43.618437808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:20:43.623313316Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:20:43.623332151Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:43.623355656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:20:43.62750525Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:20:44.238185882Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-13T18:20:44.897488569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:20:44.902827119Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T18:20:45.62226677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:46.715475296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:20:46.723204052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:20:47.490611264Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T18:20:50.60427136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:20:50.62207607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:20:50.744405526Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T18:20:51.824618178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:20:51.828527988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:20:51.828556497Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:51.828582457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:20:51.831746697Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:20:53.822387604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:20:53.824929582Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:20:53.824948146Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:20:53.824969213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:20:53.827066072Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:20:53.996684934Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T18:20:55.622746425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:20:56.24931408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:20:56.253622518Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:20:57.249670013Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-13T18:20:57.252603979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:20:57.257811915Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:20:58.54582098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:20:58.549397044Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:20:58.929628899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:20:58.934136799Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:21:00.251959781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:21:00.254605749Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:21:00.406118355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:21:00.408450619Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:21:00.472583215Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:21:00.502922846Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T18:21:00.622508683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:02.363173683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:21:02.366788572Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:21:03.75576877Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-13T18:21:05.622207085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:06.902531972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:21:06.905164092Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=debug ts=2026-05-13T18:21:07.011470005Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T18:21:07.011491627Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=9
level=info ts=2026-05-13T18:21:07.015674737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:21:07.018129412Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-13T18:21:08.101312936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:21:08.10414575Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:21:10.224930478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:21:10.22750035Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:21:10.262841069Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=info ts=2026-05-13T18:21:10.622006139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:21:10.622049195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T18:21:10.622032902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:10.622060549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:21:10.848191351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:21:10.853368265Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:21:13.514752598Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T18:21:13.549688974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:21:13.552627878Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:21:13.552646239Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:21:13.552676042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:21:13.554668668Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:21:14.156438732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:21:14.159520444Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T18:21:14.399417011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:21:14.406667457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T18:21:15.076620346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:52702"
level=info ts=2026-05-13T18:21:15.148244026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:21:15.152151706Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-13T18:21:15.622277876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:21:16.767891456Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T18:21:19.007827993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:21:19.015849296Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:21:20.021550813Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-13T18:21:20.605230635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:21:20.622011362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:21.532338036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:21:21.53590912Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:21:23.273735334Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-13T18:21:23.273759136Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T18:21:23.666490398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:21:23.67123565Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:21:25.444688882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T18:21:25.622734329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:21:26.526766253Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-13T18:21:28.033869953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:21:28.039013042Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:21:28.630765902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:21:28.63323608Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:21:28.633255275Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:21:28.633274528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:21:28.635080079Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:21:29.056293203Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:21:29.677921143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:21:29.68101011Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:21:29.78026723Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T18:21:29.953815068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:21:29.956335352Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:21:30.622742734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:21:33.033425623Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=warn ts=2026-05-13T18:21:35.622337495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:21:36.286644522Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T18:21:36.719983553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:21:36.722801084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:21:38.217068036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:21:38.219576712Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:21:38.272650443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:21:38.277106297Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:21:38.277124427Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:21:38.277148967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:21:38.282124291Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:21:39.541039055Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-13T18:21:39.541060865Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=info ts=2026-05-13T18:21:40.562654555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:21:40.568985969Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:21:40.59460538Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:21:40.598730634Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:21:40.622073622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:40.622078582Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:21:40.62209648Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:21:40.622105573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:21:41.422313572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:21:41.424844853Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:21:42.084799895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:21:42.087539082Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:21:42.792573978Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T18:21:45.622375406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:21:46.04543081Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
ts=2026-05-13T18:21:47.333759901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:49240"
level=info ts=2026-05-13T18:21:47.463681149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:21:47.466611219Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:21:49.29804144Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-13T18:21:50.603102515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:21:50.605701341Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-13T18:21:50.606836542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:21:50.622514522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:51.266685094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:21:51.270999756Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:21:52.551674046Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-13T18:21:53.966791975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:21:53.970013807Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:21:55.158460784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:21:55.161894119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:21:55.622648713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:21:55.779735236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:21:55.783100176Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:21:55.804051413Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-13T18:21:57.367801291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:21:57.370460069Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=debug ts=2026-05-13T18:21:59.057336239Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-13T18:21:59.423751561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:22:00.302152906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:22:00.306958114Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T18:22:00.622786125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:22:01.460291871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:22:01.466469804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:22:02.309849211Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T18:22:02.420238768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:22:02.423468524Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:22:05.563251423Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-13T18:22:05.62228394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:22:06.328670765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" idle=1 full=1 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:22:06.335841772Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:22:06.735926341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:22:06.738275301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:22:07.789330598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:22:07.792454454Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T18:22:08.065922551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:22:08.068296508Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:22:08.460920498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:22:08.463357221Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:22:08.816639504Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-13T18:22:10.622626189Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:22:10.622623379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:10.622653537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:22:10.622670214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:22:11.482891159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:22:11.48721459Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:22:12.069033513Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-13T18:22:12.069062875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T18:22:12.235373612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:22:12.237696276Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T18:22:15.085017127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:55010"
level=debug ts=2026-05-13T18:22:15.322634592Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-13T18:22:15.339803318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T18:22:15.622455888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:18.575407547Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
ts=2026-05-13T18:22:20.607544657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:22:20.622284844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:21.827947977Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T18:22:23.35150522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:22:23.356083568Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:22:25.081162435Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T18:22:25.622266033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:28.334976Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T18:22:29.571145509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:22:29.576732077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:22:29.878711445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:22:29.881445583Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:22:30.622317932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:31.58740704Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-13T18:22:32.237581369Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:22:33.745401028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:22:33.749075241Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:22:34.840482527Z caller=checkpoint.go:372 msg="writing series" size="686 kB"
level=warn ts=2026-05-13T18:22:35.622768004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:22:35.77743859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:22:35.784734044Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:22:36.99773802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:22:37.001319992Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:22:38.09374219Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-13T18:22:38.093764197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-13T18:22:39.9144954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:22:39.917919475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:22:40.595544985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:22:40.59864921Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:22:40.621932058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:22:40.621943151Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:22:40.621964567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:22:40.621972095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:22:40.659244265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:22:40.662222678Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:22:41.346981974Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=info ts=2026-05-13T18:22:41.914464333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:22:41.916922577Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-13T18:22:44.301143853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:22:44.304450272Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:22:44.599498802Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T18:22:45.622527765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:47.85245149Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
ts=2026-05-13T18:22:48.55736173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59202"
level=info ts=2026-05-13T18:22:49.264776906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:22:49.267940305Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T18:22:49.67109511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:22:49.675725875Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:22:50.42478645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:22:50.427338918Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:22:50.44038631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:22:50.442740828Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T18:22:50.609347018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:22:50.621935728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:51.105984799Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-05-13T18:22:52.504758024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:22:52.507626459Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=info ts=2026-05-13T18:22:53.965099523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:22:53.972971955Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:22:54.358140056Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T18:22:54.358165969Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T18:22:54.390377156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:22:54.393749317Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:22:54.417815073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:22:54.425135217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:22:55.62276968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:22:56.871150824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:22:56.872656988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T18:22:57.611298609Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=debug ts=2026-05-13T18:22:59.227759123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:22:59.374013812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:22:59.378598749Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:23:00.370541201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:23:00.375260414Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:23:00.622411079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:00.864322407Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-13T18:23:04.137432266Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-13T18:23:04.137455546Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=3
level=info ts=2026-05-13T18:23:05.308755093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:23:05.311246954Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T18:23:05.621989147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:07.370471589Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T18:23:10.622349603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:23:10.622365312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:23:10.62238947Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:23:10.622402831Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:23:10.62347033Z caller=checkpoint.go:384 msg="flushing series" totalSize="64 kB" series=1
level=info ts=2026-05-13T18:23:10.627895578Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-13T18:23:10.6291293Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006214433s
level=info ts=2026-05-13T18:23:11.441364076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:23:11.444806044Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:23:11.53303831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:23:11.536420234Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:23:14.1277939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:23:14.133159709Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T18:23:15.622299152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:23:18.559240059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42544"
level=info ts=2026-05-13T18:23:19.90316819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:23:19.906553314Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:23:20.368641865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:23:20.376004788Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:23:20.578918006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:23:20.582105693Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-13T18:23:20.61158827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:23:20.622492546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:23:20.631648112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:23:20.634224138Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:23:25.622004266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:23:27.825221292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:23:27.827918418Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:23:29.651781598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:23:29.654964455Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T18:23:30.260657915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:23:30.263806293Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=warn ts=2026-05-13T18:23:30.621941209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:30.892844861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:23:32.940531992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:23:32.944617928Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:23:34.477467572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:23:34.47996212Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-13T18:23:35.622732037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:23:36.053855805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:23:36.056395671Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:23:37.569016618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:23:37.574005812Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:23:37.641905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:23:38.673385706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:23:38.676485269Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:23:38.833004349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:23:38.836074334Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:23:39.705579566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:23:39.710759848Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:23:40.594678225Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:23:40.598789988Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:23:40.622177441Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T18:23:40.622217735Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:23:40.622218458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:40.622240523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:23:40.622251577Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:23:40.622311867Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-13T18:23:40.622323464Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T18:23:40.622346995Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.303µs success=true
level=debug ts=2026-05-13T18:23:40.622664369Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-13T18:23:40.920551771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:23:40.923283091Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=debug ts=2026-05-13T18:23:43.875862291Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-13T18:23:45.622397137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:47.129485322Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T18:23:47.550555803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:23:47.557557572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T18:23:48.561178058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50996"
level=info ts=2026-05-13T18:23:48.834668698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:23:48.842634329Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:23:50.381749335Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
ts=2026-05-13T18:23:50.613245985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:23:50.62191742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:53.635422257Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-13T18:23:54.495649966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:23:54.498866562Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:23:55.435580596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:23:55.438231256Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:23:55.622223561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:23:56.888921051Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=debug ts=2026-05-13T18:23:56.888941612Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-13T18:24:00.141485471Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-13T18:24:00.622840699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:03.276456921Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:24:03.394628959Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T18:24:03.956519046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:24:03.96098238Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:24:04.266802609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:24:04.269591716Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:24:05.523657508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:24:05.52680628Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:24:05.622068058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:24:06.57215674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:24:06.575268961Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:24:06.646747188Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T18:24:07.171984219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:24:07.178147749Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:24:08.757680699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:24:08.76090308Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:24:09.900617631Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T18:24:09.900640135Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T18:24:10.622101493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:24:10.622146308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:24:10.62215499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:24:10.622169707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:13.153405182Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-13T18:24:15.091004216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:35144"
level=info ts=2026-05-13T18:24:15.196843237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:24:15.199774882Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T18:24:15.622550133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:24:16.018595661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:24:16.024910678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:24:16.02492872Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:24:16.02495892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:24:16.031395648Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:24:16.405970537Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=info ts=2026-05-13T18:24:17.040496717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:24:17.044161983Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-13T18:24:17.046212394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:24:17.048695014Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-13T18:24:17.34261029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:33710"
ts=2026-05-13T18:24:18.563177547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56450"
level=info ts=2026-05-13T18:24:19.345130071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:24:19.348135909Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:24:19.348153909Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:24:19.348174316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:24:19.350960204Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:24:19.659268545Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=debug ts=2026-05-13T18:24:19.659297916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-13T18:24:20.614429379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:24:20.622241715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:22.914701168Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:24:22.914721693Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=info ts=2026-05-13T18:24:24.201707383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:24:24.204489055Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:24:24.645780932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:24:24.64885943Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:24:24.873487229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:24:24.876191612Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:24:25.622528802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:26.165490325Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T18:24:27.250793108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:24:27.255836567Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:24:27.260850107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=674aadd9971b1f92 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="401 kB" avg_uncomp="401 kB" full=1 labels="{stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:24:27.263472683Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/674aadd9971b1f92/19e224c1db1%3A19e2257d1c9%3Af5022c3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-psft\"}"
level=error ts=2026-05-13T18:24:27.26349884Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:24:27.263531225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="203 kB" avg_uncomp="203 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:24:27.265923051Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T18:24:28.972860476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:24:28.979737389Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:24:29.159281836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:24:29.166264463Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:24:29.46325107Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-13T18:24:29.46327895Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=2
level=debug ts=2026-05-13T18:24:29.612728166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:24:30.552215696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:24:30.560253804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T18:24:30.622796973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:30.702209243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:24:31.729889635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:24:31.733749515Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:24:32.671503783Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T18:24:35.622892761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:24:35.657038229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:24:35.662307847Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:24:35.925896498Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T18:24:35.925919672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T18:24:37.237542415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:24:37.24094288Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:24:37.240963355Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:24:37.240984898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:24:37.243864622Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:24:39.177807367Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-13T18:24:40.595049776Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:24:40.599156004Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:24:40.622565185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:24:40.622565021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:40.622589095Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:24:40.622598833Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:24:42.431407932Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-13T18:24:42.431436223Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-13T18:24:44.518380056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:24:44.52105678Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:24:45.561618449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:24:45.564160183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T18:24:45.622272499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:45.683504164Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-13T18:24:48.565001413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54090"
level=debug ts=2026-05-13T18:24:48.937896082Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T18:24:48.937918423Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T18:24:49.198552769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:24:49.204323281Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:24:49.230406646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:24:49.234110903Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T18:24:50.616301818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:24:50.622385089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:24:51.293073492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:24:51.296340328Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:24:52.189153127Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:24:52.368258331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:24:52.371292113Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:24:52.462635969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:24:52.466277268Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-13T18:24:52.466294869Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:24:52.466314766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:24:52.469503852Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:24:53.140599497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:24:53.147537251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:24:53.298930152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:24:53.304231634Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:24:55.442467295Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-13T18:24:55.622155569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:24:58.696467799Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T18:24:58.696490055Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-13T18:25:00.621932979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:00.999145256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:25:01.131319036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:25:01.134225301Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:25:01.591751245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:25:01.595251709Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:25:01.595267758Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:25:01.595287262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:25:01.598184377Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:25:01.948210401Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-13T18:25:05.201285816Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T18:25:05.622057297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:25:05.696363821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:25:05.700893781Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:25:06.857081966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:25:06.860061092Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:25:07.367203753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:25:07.370846364Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:25:07.443192569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:25:07.44611607Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:25:08.456352253Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-13T18:25:08.456375227Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T18:25:10.622555528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:25:10.622555582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:10.622581166Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:25:10.622589736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:25:10.79126905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:25:10.794808099Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:25:10.873890128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:25:10.876259836Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:25:11.640784171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:25:11.644364439Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T18:25:11.707793607Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=info ts=2026-05-13T18:25:13.995978778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="52 kB" avg_uncomp="52 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:25:13.9986743Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e2242310c%3A19e22591324%3Ace80135f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T18:25:13.998693878Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:25:13.998713502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="478 kB" avg_uncomp="478 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:25:14.001390908Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T18:25:14.591493195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:25:14.594761195Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:25:14.960561661Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
ts=2026-05-13T18:25:15.093570542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:35164"
level=info ts=2026-05-13T18:25:15.142912594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:25:15.146864675Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T18:25:15.622316734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:16.636357149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T18:25:17.345402528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:53892"
level=debug ts=2026-05-13T18:25:18.21354734Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-13T18:25:18.472159552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:25:18.478418772Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:25:18.478444478Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:25:18.47846796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:25:18.48623194Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:25:18.851756282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:25:18.855895265Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T18:25:20.617808862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:25:20.622520299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:25:20.898773103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:25:20.904897021Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:25:21.466017267Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T18:25:24.23253873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:25:24.235549828Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:25:24.602763103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:25:24.609041296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:25:24.720231761Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T18:25:24.720253261Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=5
level=warn ts=2026-05-13T18:25:25.622559274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:27.972295051Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=info ts=2026-05-13T18:25:29.210732023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:25:29.215587732Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T18:25:29.870644212Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:25:30.621906141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:31.225289164Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-13T18:25:34.47866884Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-13T18:25:35.529936119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:25:35.537104964Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T18:25:35.622262257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:37.732351802Z caller=checkpoint.go:372 msg="writing series" size="726 kB"
level=info ts=2026-05-13T18:25:38.766082474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:25:38.769197178Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:25:40.595050406Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:25:40.599163157Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T18:25:40.62253522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:25:40.622562078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:25:40.622580312Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:25:40.622587932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:25:40.947457044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:25:40.95142581Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:25:40.984327667Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T18:25:40.984343844Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T18:25:41.200244195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:25:41.202590844Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:25:42.078170949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:25:42.082369162Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:25:44.103553874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:25:44.106079069Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:25:44.106098027Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:25:44.106116965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:25:44.108076789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T18:25:44.237592564Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
ts=2026-05-13T18:25:45.094395419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:37322"
level=warn ts=2026-05-13T18:25:45.621922842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:25:45.813223492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:25:45.815642711Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:25:46.340365542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:25:46.345131378Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T18:25:47.34672555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:60564"
level=debug ts=2026-05-13T18:25:47.490815868Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T18:25:48.569028771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58346"
level=info ts=2026-05-13T18:25:50.212623392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:25:50.218531775Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T18:25:50.618461531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=warn ts=2026-05-13T18:25:50.622205133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:25:50.743546787Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T18:25:51.187413343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:25:51.19089399Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:25:53.549156033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:25:53.552285169Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:25:53.996906274Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T18:25:55.161016976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:25:55.163635616Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:25:55.16365889Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:25:55.163684131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:25:55.165956094Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:25:55.62016223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T18:25:55.621998307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-13T18:25:55.623037226Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:25:56.538407199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:25:56.546571161Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:25:56.938934726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:25:56.941894602Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T18:25:57.249944975Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-13T18:25:59.882423944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:25:59.886321972Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:26:00.388756029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:26:00.391491996Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T18:26:00.502266257Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T18:26:00.622695667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:26:01.716912427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:26:01.921418431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:26:01.924775241Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:26:02.375929424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:26:02.381410632Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:26:03.755511993Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T18:26:04.297011383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:26:04.300781004Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:26:04.300811405Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:26:04.300837097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:26:04.303842127Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:26:05.420776932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="397 kB" avg_comp="397 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:26:05.424689694Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:26:05.424728501Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:26:05.424768135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:26:05.427939586Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-13T18:26:05.622391139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:06.251378351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:26:06.254405967Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:26:07.011655454Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T18:26:07.011683553Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=8
level=info ts=2026-05-13T18:26:09.277283193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:26:09.280850766Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:26:10.262826908Z caller=checkpoint.go:372 msg="writing series" size="630 kB"
level=info ts=2026-05-13T18:26:10.622398873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:26:10.622435955Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:26:10.622455529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:26:10.622467738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:12.341091754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:26:12.34756137Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:26:13.514671809Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-13T18:26:13.699854648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T18:26:15.622656079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:16.093466265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:26:16.101895256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:26:16.768032083Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-13T18:26:16.768063152Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-13T18:26:17.067845873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:26:17.071574006Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
ts=2026-05-13T18:26:17.34842196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:45422"
level=debug ts=2026-05-13T18:26:20.020208318Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-13T18:26:20.619365319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=warn ts=2026-05-13T18:26:20.622236742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:22.781023086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:26:22.789710424Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T18:26:23.274309095Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-13T18:26:25.622772713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:25.797677163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:26:25.801228683Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:26:25.931244268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:26:25.934343375Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:26:26.526503048Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-13T18:26:28.395478079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:26:28.399863492Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:26:28.98229099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:26:28.985292573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:26:29.779582251Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T18:26:30.241192902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:26:30.246210189Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T18:26:30.622324561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:26:33.033096383Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-13T18:26:34.143560332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:26:34.148026797Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:26:35.502558596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:26:35.506686778Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:26:35.567542185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:26:35.573223792Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T18:26:35.622535793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:26:36.286269401Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-13T18:26:36.388715213Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:26:37.45768667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:26:37.461350047Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T18:26:39.542060775Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-13T18:26:39.542091604Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=7
level=debug ts=2026-05-13T18:26:40.59553094Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:26:40.598659744Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:26:40.621990513Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:26:40.622036671Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:26:40.622053112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:26:40.622054939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:41.598005864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:26:41.601744388Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:26:42.211590871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:26:42.219952206Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:26:42.791806706Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-13T18:26:43.490401579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:26:43.493334069Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T18:26:45.622555743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:26:46.046354307Z caller=checkpoint.go:372 msg="writing series" size="423 kB"
ts=2026-05-13T18:26:47.350373782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:54720"
level=debug ts=2026-05-13T18:26:49.29809036Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-13T18:26:50.500713792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:26:50.504137013Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-13T18:26:50.621619358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=warn ts=2026-05-13T18:26:50.621977806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:26:52.073853398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:26:52.077557938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:26:52.077587924Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:26:52.077615547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:26:52.079943332Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:26:52.120151547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:26:52.12372674Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:26:52.551137328Z caller=checkpoint.go:372 msg="writing series" size="287 kB"
level=info ts=2026-05-13T18:26:53.78047924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:26:53.788083531Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:26:53.796098956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:26:53.799217296Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=warn ts=2026-05-13T18:26:55.622908768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:26:55.80419919Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-13T18:26:55.960708748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:26:55.964221702Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T18:26:58.111386897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:26:58.115605846Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T18:26:59.056376999Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-13T18:27:00.622237492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:27:02.218276547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:27:02.223866107Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:27:02.223907131Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:02.22393768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:27:02.228097223Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:27:02.310386359Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T18:27:03.050787652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:27:04.647593764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:27:04.650444177Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:27:05.563143291Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-13T18:27:05.563168946Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-13T18:27:05.622728969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:27:07.8675453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:27:07.873476877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:27:07.873504409Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:07.873533788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:27:07.878452593Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:27:08.816412202Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-13T18:27:09.427185766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:27:09.431304724Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:27:09.431332799Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:09.431362384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:27:09.434781498Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:27:10.622507919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:27:10.622528524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:27:10.622558475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:27:10.622570024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:27:12.069157863Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-13T18:27:12.165190836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:27:12.166946237Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T18:27:12.569234085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:27:12.575965606Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:27:12.575994428Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:12.576042626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:27:12.582470954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:27:13.491826429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:27:13.494708553Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:27:13.494741112Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:13.49476909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:27:13.497164769Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:27:13.895507675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:27:13.898622341Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:27:15.32191182Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T18:27:15.622446685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:27:15.77913573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:27:15.784787182Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:27:17.49146172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:27:17.495360518Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:27:17.798481878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:27:17.801562307Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T18:27:18.574751081Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
ts=2026-05-13T18:27:18.575654588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36274"
level=info ts=2026-05-13T18:27:19.689362598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:27:19.693278517Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T18:27:20.622542827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:27:20.623101887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=info ts=2026-05-13T18:27:21.226286232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:27:21.231575536Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:27:21.82837949Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T18:27:23.211169927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:27:23.216158137Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:27:24.871785621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:27:24.876103713Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:27:24.876132911Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:24.87616323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:27:24.878597339Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T18:27:25.081281621Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T18:27:25.622404724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:27:27.439669888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:27:27.442760146Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:27:27.442784513Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:27:27.442818377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:27:27.444995394Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:27:27.564283021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:27:27.567079947Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:27:28.334654293Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T18:27:29.318414059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:27:29.324982981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:27:30.622003331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:27:31.296079317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:27:31.586917257Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T18:27:32.128488899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:27:32.135495736Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:27:32.387348355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:27:32.389969984Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:27:32.576203424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:27:32.578590868Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:27:34.630908833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:27:34.634394946Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:27:34.840348366Z caller=checkpoint.go:372 msg="writing series" size="686 kB"
level=debug ts=2026-05-13T18:27:34.840374226Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-05-13T18:27:35.622601312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:27:36.2425538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:27:36.245296695Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T18:27:36.495011095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:27:36.498170489Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:27:38.093074022Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=info ts=2026-05-13T18:27:38.62853139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:27:38.631515207Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:27:38.89916528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:27:38.901691935Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:27:39.432182968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:27:39.435246671Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T18:27:40.594893915Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:27:40.598956117Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:27:40.62234469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:27:40.622383122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:27:40.622398817Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:27:40.62239478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:27:41.346522658Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=info ts=2026-05-13T18:27:43.203806992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:27:43.210366961Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:27:43.917349863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:27:43.922493468Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:27:44.412017387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:27:44.41539383Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:27:44.599068691Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T18:27:45.622576422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:27:46.609723375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:27:46.616317953Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:27:47.851836367Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=debug ts=2026-05-13T18:27:47.851857823Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T18:27:50.622825816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:27:50.625219387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=debug ts=2026-05-13T18:27:50.689264598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:27:50.91953003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:27:50.922309681Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T18:27:51.105179295Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-13T18:27:51.303575717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:27:51.308675399Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:27:54.358587842Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-05-13T18:27:55.622505641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:27:57.611378335Z caller=checkpoint.go:372 msg="writing series" size="391 kB"
level=warn ts=2026-05-13T18:28:00.622326385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:28:00.864547294Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T18:28:01.724566001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:28:01.730732728Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:28:01.778861923Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:28:03.01963949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:28:03.025983259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:28:04.139531087Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-13T18:28:04.139562078Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=5
level=warn ts=2026-05-13T18:28:05.622009446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:06.876721615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:28:06.882806123Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:28:06.882831784Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:28:06.882863914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:28:06.887933138Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:28:07.37022529Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T18:28:07.483613011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="15 MB" avg_comp="2.2 MB" total_uncomp="135 MB" avg_uncomp="19 MB" full=5 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:28:07.491714993Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:28:08.323355443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:28:08.326906742Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:28:08.451644342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:28:08.454698289Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:28:08.877935261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:28:08.880559062Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T18:28:09.29183989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:28:09.294615435Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T18:28:10.621972752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:10.62201508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:28:10.62205013Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:28:10.622058374Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T18:28:10.623073742Z caller=checkpoint.go:384 msg="flushing series" totalSize="64 kB" series=1
level=info ts=2026-05-13T18:28:10.63337932Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-13T18:28:10.63590522Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013402544s
level=info ts=2026-05-13T18:28:13.193408528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:28:13.196214712Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:28:13.196243924Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:28:13.196273092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:28:13.198396521Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:28:15.622538132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:16.07183033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:28:16.074463634Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T18:28:18.578696047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53038"
level=warn ts=2026-05-13T18:28:20.621995872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:28:20.626295198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=info ts=2026-05-13T18:28:21.922079291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:28:21.925829233Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T18:28:23.077036372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:28:23.08226689Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T18:28:23.08737217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:28:23.090314999Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:28:24.954217613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:28:24.959713476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:28:25.035136149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:28:25.038036602Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T18:28:25.622529791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:28:25.891322057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:28:26.562392046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:28:26.565273154Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T18:28:29.127496478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:28:29.135572341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:28:29.99724856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:28:30.00059918Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-13T18:28:30.622051546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:33.521168929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:28:33.526122396Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:28:33.631519122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:28:33.634407897Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T18:28:35.44866665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:28:35.451909989Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T18:28:35.621994613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:35.642091639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:28:35.64602216Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:28:39.564205384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:28:39.568180953Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:28:40.594937819Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:28:40.599046677Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:28:40.622426847Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T18:28:40.622459446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:28:40.622483743Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:28:40.622493859Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:28:40.62256085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:40.622581755Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-13T18:28:40.622591236Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T18:28:40.622610269Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.453µs success=true
level=debug ts=2026-05-13T18:28:40.622975893Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-13T18:28:40.638100757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:28:40.645663303Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:28:42.512683133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:28:43.876572261Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-13T18:28:45.105607717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:44322"
level=warn ts=2026-05-13T18:28:45.622502677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:28:47.129116274Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-13T18:28:47.937999868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:28:47.967552767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T18:28:48.489171447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:28:48.49433603Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T18:28:48.580727719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59582"
level=info ts=2026-05-13T18:28:48.70755788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:28:48.710137707Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:28:50.382944721Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=warn ts=2026-05-13T18:28:50.62251669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:28:50.62817984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-601f363a 10.130.0.54:7946"
level=debug ts=2026-05-13T18:28:51.821517469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:28:53.635490107Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-13T18:28:55.622505823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:28:56.160161939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:28:56.163278512Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:28:56.360561559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:28:56.363691149Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:28:56.889101605Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=debug ts=2026-05-13T18:28:56.889128157Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-13T18:29:00.141181552Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-13T18:29:00.622559682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:03.39568025Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=info ts=2026-05-13T18:29:03.434991663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:29:03.438105691Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T18:29:05.622230802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:29:05.644393187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:29:05.647952102Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T18:29:05.675946448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:29:05.679031304Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-13T18:29:06.647910349Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T18:29:06.759095207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:29:06.762436254Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:29:07.285929943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.3 MB" total_uncomp="158 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:29:07.293154334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T18:29:08.990918438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:29:08.994207125Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:29:09.901131298Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=debug ts=2026-05-13T18:29:09.901155291Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T18:29:10.198569962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:29:10.202175942Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=info ts=2026-05-13T18:29:10.621870839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:29:10.621933483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:29:10.621944172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:29:10.6219365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:13.153167246Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T18:29:15.622237482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:16.406855819Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
ts=2026-05-13T18:29:17.35911941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:41172"
ts=2026-05-13T18:29:18.582510167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56278"
level=info ts=2026-05-13T18:29:19.081821293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:29:19.08771034Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T18:29:19.659691615Z caller=checkpoint.go:372 msg="writing series" size="718 kB"
level=debug ts=2026-05-13T18:29:19.659718403Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-13T18:29:20.622848113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:29:20.63015868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=info ts=2026-05-13T18:29:20.95840131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:29:20.961624899Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T18:29:22.052849578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:29:22.056056523Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:29:22.719077131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:29:22.722852937Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:29:22.726917704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:29:22.730196826Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T18:29:22.914900714Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:29:22.914924403Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=debug ts=2026-05-13T18:29:22.977054569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T18:29:23.386116797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:29:23.389035565Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T18:29:25.622788848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:26.165540979Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T18:29:27.357831926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:29:27.365176813Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T18:29:28.083161237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:29:28.086730325Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T18:29:29.478237668Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T18:29:29.478270753Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=2
level=warn ts=2026-05-13T18:29:30.621939145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:29:31.279197255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:29:31.282542344Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:29:31.804222399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:29:31.813315025Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=debug ts=2026-05-13T18:29:32.671586947Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-13T18:29:35.622461915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:35.92551838Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T18:29:35.925544095Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-13T18:29:36.453084472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T18:29:39.177825098Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=info ts=2026-05-13T18:29:40.257652247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:29:40.260174858Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:29:40.594850046Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:29:40.5989804Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T18:29:40.622362577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T18:29:40.622375637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:40.622389895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:29:40.622400758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:29:41.860493794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:29:41.867249242Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:29:42.431333955Z caller=checkpoint.go:372 msg="writing series" size="529 kB"
level=debug ts=2026-05-13T18:29:42.43135742Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-13T18:29:44.213222921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:29:44.219395771Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T18:29:45.622938397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:29:45.683208144Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-13T18:29:48.9377121Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T18:29:48.937736243Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-13T18:29:49.239562275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:29:49.244195614Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T18:29:50.62204553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:29:50.63143766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=debug ts=2026-05-13T18:29:52.189967429Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-13T18:29:52.644291229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:29:52.647438408Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:29:52.721616191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:29:52.725000543Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:29:55.442907598Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-13T18:29:55.502081578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.3 MB" total_uncomp="158 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:29:55.510174502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:29:55.53035133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T18:29:55.622000725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:29:56.761933716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:29:56.764594648Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T18:29:57.729712368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:29:57.732308355Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:29:57.820613832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:29:57.823890596Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:29:58.696783916Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T18:29:58.6968056Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-05-13T18:29:59.072364931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:29:59.075770692Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:29:59.740656751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:29:59.745645511Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T18:30:00.622425993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:30:01.948972722Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T18:30:02.947229243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:30:02.950260126Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T18:30:03.017614398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:30:03.020458937Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T18:30:04.926161219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:30:04.930121058Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:30:05.202031269Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T18:30:05.622806078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:30:07.517184393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:30:07.520515644Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T18:30:08.271596161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:30:08.274242598Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T18:30:08.457468087Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T18:30:08.457490944Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T18:30:10.621828495Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:30:10.621866163Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:30:10.621895819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:30:10.621948418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:30:11.708384789Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=debug ts=2026-05-13T18:30:14.961004889Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T18:30:15.556713181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:30:15.5623017Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T18:30:15.622371638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:30:15.851393323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:30:15.859091084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T18:30:18.214257923Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-13T18:30:19.269515884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:30:19.274451108Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T18:30:20.622356601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:30:20.632760052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-4lqrn-1019306d 10.128.0.98:7946"
level=debug ts=2026-05-13T18:30:21.466631965Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T18:30:24.359374161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:30:24.362983453Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:30:24.490533984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:30:24.495970905Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:30:24.721170526Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T18:30:24.721194753Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=info ts=2026-05-13T18:30:24.863892449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b3c3b0e59091ed43 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="209 kB" avg_uncomp="209 kB" idle=1 labels="{stream=\"build-e2e-rwkz\"}"
level=debug ts=2026-05-13T18:30:24.86391886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T18:30:24.867455172Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b3c3b0e59091ed43/19e224d0534%3A19e226366a1%3Ad64ef053?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rwkz\"}"
level=info ts=2026-05-13T18:30:25.618349499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:30:25.621722778Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T18:30:25.622802833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:30:27.729909272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:30:27.735771453Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T18:30:27.972431922Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-13T18:30:30.390290116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:30:30.397831128Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=warn ts=2026-05-13T18:30:30.622640439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:30:31.226370649Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-13T18:30:33.755684266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T18:30:34.018746234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:30:34.021492439Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T18:30:34.47894825Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-13T18:30:35.622535388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:30:37.732980879Z caller=checkpoint.go:372 msg="writing series" size="881 kB"
level=debug ts=2026-05-13T18:30:37.733006819Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T18:30:39.149991543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b3c3b0e59091ed43 immediate=false num_chunks=1 total_comp="57 kB" avg_comp="57 kB" total_uncomp="209 kB" avg_uncomp="209 kB" full=1 labels="{stream=\"build-e2e-rwkz\"}"
level=error ts=2026-05-13T18:30:39.153263648Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b3c3b0e59091ed43/19e224d0534%3A19e226366a1%3Ad64ef053?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rwkz\"}"
level=debug ts=2026-05-13T18:30:40.594887272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.100:11211 count=1
level=info ts=2026-05-13T18:30:40.599003711Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T18:30:40.599206466Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T18:30:40.599250093Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976329 n=1
level=debug ts=2026-05-13T18:30:40.599289067Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976329 n=1
level=info ts=2026-05-13T18:30:40.622338166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:30:40.622366134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T18:30:40.622355242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:30:40.622377184Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T18:30:40.662708033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.3 MB" total_uncomp="158 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:30:40.670221146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T18:30:40.984911553Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-13T18:30:42.026930975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T18:30:42.029638732Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2242ab32%3A19e2242ab33%3A95decad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T18:30:44.237922021Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-13T18:30:45.280797632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:30:45.28395229Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T18:30:45.62257276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:30:46.269579212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="483 kB" avg_comp="483 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-13T18:30:46.273471465Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e22421baf%3A19e22664389%3Abea7cace?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-13T18:30:46.948489355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:30:46.951681542Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:30:47.490644174Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T18:30:48.588675321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52734"
level=warn ts=2026-05-13T18:30:50.622852627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:30:50.634156561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=info ts=2026-05-13T18:30:50.655834595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T18:30:50.659133332Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e22422bcb%3A19e2259076d%3A2fd2d40d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T18:30:50.743326819Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T18:30:51.406447218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T18:30:51.409222512Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2242ab31%3A19e2242ab33%3A85d210c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T18:30:53.528444443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:30:53.531294619Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:30:53.531316366Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:30:53.531339911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:30:53.53361033Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T18:30:53.997199727Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T18:30:54.534489639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T18:30:54.539489937Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e22421c96%3A19e226dd78d%3Ad03607f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T18:30:54.731557286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="478 kB" avg_uncomp="478 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T18:30:54.734415193Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e22421e88%3A19e2259d008%3A949bc380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T18:30:55.622146023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:30:56.810775322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=18a8b49676e03338 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-hcua\"}"
level=error ts=2026-05-13T18:30:56.814010659Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/18a8b49676e03338/19e224d0bf8%3A19e2259121a%3Aad0c0716?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-hcua\"}"
level=debug ts=2026-05-13T18:30:57.250350008Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-13T18:30:57.329482721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:31:00.503421799Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T18:31:00.62251453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:31:01.497385193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T18:31:01.50041722Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2242962c%3A19e2242b11a%3A181f0e0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T18:31:03.756460834Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-13T18:31:04.323100215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="397 kB" avg_comp="397 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T18:31:04.326375133Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e22422c5a%3A19e228cd419%3A6b8bddc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-13T18:31:05.62235676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:31:07.010556208Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-13T18:31:07.010578512Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=9
level=info ts=2026-05-13T18:31:07.56311654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b3c3b0e59091ed43 immediate=false num_chunks=1 total_comp="57 kB" avg_comp="57 kB" total_uncomp="209 kB" avg_uncomp="209 kB" full=1 labels="{stream=\"build-e2e-rwkz\"}"
level=error ts=2026-05-13T18:31:07.566568609Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b3c3b0e59091ed43/19e224d0534%3A19e226366a1%3Ad64ef053?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rwkz\"}"
level=info ts=2026-05-13T18:31:08.879407342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:31:08.883407593Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:31:08.883429962Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:31:08.883455822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:31:08.886734551Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:31:10.084206722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T18:31:10.086967885Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2242962c%3A19e2242c81c%3A936e14e8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T18:31:10.261799143Z caller=checkpoint.go:372 msg="writing series" size="539 kB"
level=info ts=2026-05-13T18:31:10.621818946Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T18:31:10.621854846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T18:31:10.621864157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T18:31:10.621931042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:31:12.352892907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="203 kB" avg_uncomp="203 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T18:31:12.355948675Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e22422ee2%3A19e2258fbb4%3A3c062510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T18:31:12.711208895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:31:12.71536267Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:31:12.715394437Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:31:12.715432727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:31:12.718864239Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T18:31:13.515453812Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T18:31:14.472843607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:31:14.479229472Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:31:14.479253274Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:31:14.479296096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:31:14.485420715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T18:31:15.114476966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.95:38384"
level=warn ts=2026-05-13T18:31:15.621995555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:31:16.545704062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.2 MB" total_uncomp="16 MB" avg_uncomp="8.2 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T18:31:16.551992565Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e2242ab31%3A19e22612481%3Ad5485779?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T18:31:16.768048384Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
ts=2026-05-13T18:31:17.364419527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.98:45456"
level=info ts=2026-05-13T18:31:17.656088202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T18:31:17.659742657Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e22421b8e%3A19e2280780a%3A4e39a5b0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T18:31:18.045933739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.2 MB" avg_comp="1.6 MB" total_uncomp="21 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T18:31:18.050931916Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e22421cc0%3A19e2255694d%3A4fe1e51e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T18:31:18.590728177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59628"
level=debug ts=2026-05-13T18:31:20.024402095Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-13T18:31:20.622765125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T18:31:20.63513609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j6q5q-16e3f244 10.128.0.95:7946"
level=debug ts=2026-05-13T18:31:23.274078039Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=debug ts=2026-05-13T18:31:23.274100122Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T18:31:24.328462719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T18:31:24.331340063Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e224c8ef9%3A19e224dcb63%3Ab82525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T18:31:24.603982023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92d3c07d1e3b891d immediate=false num_chunks=2 total_comp="3.1 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="6.1 MB" full=2 labels="{stream=\"group-rjhz\"}"
level=error ts=2026-05-13T18:31:24.611135449Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92d3c07d1e3b891d/19e224d5e00%3A19e2259b899%3A4f7d3d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rjhz\"}"
level=info ts=2026-05-13T18:31:24.984990395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.3 MB" total_uncomp="158 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T18:31:24.991959303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e224223b4%3A19e2242b06e%3Afccd12f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T18:31:25.622852745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:31:26.526681125Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-13T18:31:26.743531387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="470 kB" avg_comp="470 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T18:31:26.747402324Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e22421a69%3A19e22808b92%3A4c70ce83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T18:31:27.614799441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="516 kB" avg_comp="516 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T18:31:27.618471413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e22421d44%3A19e22566509%3A34b66e9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T18:31:29.680573391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T18:31:29.685708349Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2242ab32%3A19e2242ab33%3A22d3dfda?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T18:31:29.779829031Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T18:31:30.044829723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="429 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T18:31:30.05176832Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e22422242%3A19e224c5295%3Ab752c7b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T18:31:30.622504493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T18:31:31.98313141Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T18:31:33.033438168Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-13T18:31:33.646339215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T18:31:33.648006313Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T18:31:35.622890293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T18:31:35.693226113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:31:35.696545374Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:31:35.696564384Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T18:31:35.696584987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="432 kB" avg_comp="432 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T18:31:35.699006563Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e22422b44%3A19e2287657a%3A93df7819?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T18:31:36.286010312Z caller=checkpoint.go:372 msg="writing series" size="18 kB"