level=info ts=2026-05-09T14:45:46.975279403Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-09T14:45:46.975312795Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-09T14:45:46.976517998Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-09T14:45:46.977310439Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-09T14:45:46.977672355Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T14:45:46.977760053Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.64 inf=eth0
level=info ts=2026-05-09T14:45:46.978325826Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-6f2a52d4
ts=2026-05-09T14:45:46.978383045Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-09T14:45:46.97839273Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.64 advertisePort=7946
level=debug ts=2026-05-09T14:45:46.978408452Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.64 advertisePort=7946
level=info ts=2026-05-09T14:45:46.978537263Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-09T14:45:46.983066332Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T14:45:46.983082487Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:45:46.983093646Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.557399ms
level=error ts=2026-05-09T14:45:46.983104705Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-09T14:45:46.983118969Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-05-09T14:45:46.983291629Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T14:45:46.98331992Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-09T14:45:46.984444838Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-09T14:45:46.984472833Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T14:45:46.985074428Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-09T14:45:46.9850938Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-09T14:45:46.985092329Z
level=debug ts=2026-05-09T14:45:46.985113197Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-09T14:45:46.985129863Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-09T14:45:46.985146585Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-09T14:45:46.985151461Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-09T14:45:46.985150524Z
level=debug ts=2026-05-09T14:45:46.98522171Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-05-09T14:45:46.986527501Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:46.98654127Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T14:45:46.98670076Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T14:45:46.986767069Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.64 inf=eth0
level=debug ts=2026-05-09T14:45:47.000743755Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-05-09T14:45:47.000743685Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-09T14:45:47.000769876Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-09T14:45:47.000768463Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-09T14:45:47.00078275Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-09T14:45:47.000797192Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-09T14:45:47.000795583Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-09T14:45:47.000828949Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-05-09T14:45:47.000838547Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-09T14:45:47.000804665Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-05-09T14:45:47.000870175Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-09T14:45:47.000868198Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-09T14:45:47.001055092Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-09T14:45:47.001057874Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-09T14:45:47.001065205Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-05-09T14:45:47.001070287Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-09T14:45:47.001081005Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-09T14:45:47.001096441Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-09T14:45:47.001118745Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-09T14:45:47.001122547Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-09T14:45:47.001125362Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-09T14:45:47.001235463Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-09T14:45:47.001239571Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-09T14:45:47.001242568Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-05-09T14:45:47.001245148Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-09T14:45:47.001261692Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-09T14:45:47.001268692Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T14:45:47.001306188Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.631µs success=true
level=info ts=2026-05-09T14:45:47.001318047Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-09T14:45:47.001376171Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=111.064µs errors=false
level=info ts=2026-05-09T14:45:47.001385187Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-09T14:45:47.001497363Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=236.73µs errors=false
level=info ts=2026-05-09T14:45:47.00150367Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-09T14:45:47.00151088Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=249.994µs
level=info ts=2026-05-09T14:45:47.001547587Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-09T14:45:47.001563478Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-09T14:45:47.001569517Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-09T14:45:47.001571087Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-09T14:45:47.001579994Z caller=loki.go:615 msg="Loki started" startup_time=94.700677ms
level=info ts=2026-05-09T14:45:47.001601627Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=2.017520363s
level=debug ts=2026-05-09T14:45:47.0016482Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-09T14:45:47.001653626Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-09T14:45:47.001728606Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-09T14:45:48.366365204Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:48.366386751Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:45:50.408512088Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:50.408534941Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:45:52.001948387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:45:56.74567229Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:45:56.745695794Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:45:57.001800447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:46:02.00192954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:46:03.238648038Z caller=logging.go:137 msg="GET /ready (503) 94.258µs"
level=warn ts=2026-05-09T14:46:07.002118811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:46:10.675798688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.95:7946"
ts=2026-05-09T14:46:10.67857526Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.98:7946: dial tcp 10.129.0.98:7946: connect: connection refused"
ts=2026-05-09T14:46:10.679338056Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.65:7946: dial tcp 10.130.0.65:7946: connect: connection refused"
ts=2026-05-09T14:46:10.679931542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.66:7946"
ts=2026-05-09T14:46:10.681386817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-05-09T14:46:10.682853606Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
level=info ts=2026-05-09T14:46:10.682863104Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.699737355s
level=warn ts=2026-05-09T14:46:12.002439289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:46:13.237679863Z caller=logging.go:137 msg="GET /ready (503) 73.337µs"
level=info ts=2026-05-09T14:46:17.001976865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:46:17.00196601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:46:17.002002554Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-09T14:46:17.00201243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:46:22.002105255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:46:23.117379501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:46:23.507331197Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T14:46:23.507375144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:46:23.513778457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:46:27.002366596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:46:32.002180511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:46:32.687554229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34342"
level=info ts=2026-05-09T14:46:35.350042028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:46:35.355654081Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:46:37.001793173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:46:42.002183032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:46:42.699765759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T14:46:44.476632404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:40306"
level=info ts=2026-05-09T14:46:46.984665411Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:46:46.985063867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:46:47.002431721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:46:47.002448919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T14:46:47.002441424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:46:47.002456479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:46:52.001791164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:46:53.119282566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:46:54.575890006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:46:57.002477221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:46:58.020788766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T14:46:59.431073405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:46:59.438931838Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:47:02.00237315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:47:07.002027455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:47:12.002240785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:47:17.002151439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:47:17.002163211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:47:17.002179325Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:47:17.002189243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:47:21.139277843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:47:22.00200011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:47:23.120833733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:47:23.421832466Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:47:27.002085879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:47:32.002691014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:47:32.691448611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41194"
level=warn ts=2026-05-09T14:47:37.002463828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:47:41.629553086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:47:42.00204068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:47:45.928298514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45158"
level=info ts=2026-05-09T14:47:46.984746765Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:47:46.985191985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:47:47.002540127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:47:47.002556509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:47:47.002564175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:47:47.002569503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:47:52.002295658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:47:53.122831211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:47:55.692708506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:47:57.001845337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:47:58.596389574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:47:58.603596709Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:48:02.001948257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:48:02.693465591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45976"
level=warn ts=2026-05-09T14:48:07.001899938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:48:08.016925962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T14:48:08.01860323Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T14:48:12.001916388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:48:15.929796766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:46902"
level=info ts=2026-05-09T14:48:17.002312543Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:48:17.002337353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T14:48:17.002332962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:48:17.002346012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:48:22.002194584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:48:23.124916628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:48:24.12875449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:48:27.002620441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:48:32.001840453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:48:37.001671145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:48:42.001986203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:48:46.453903551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T14:48:46.984822134Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:48:46.985268079Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:48:47.002609387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:48:47.00261918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:48:47.002628155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:48:47.002637806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:48:50.809913818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:48:52.001749821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:48:52.022853197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:48:52.029105357Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T14:48:53.127169633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=warn ts=2026-05-09T14:48:57.002084966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:49:02.0026114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:49:07.001638215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:49:12.002637933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:49:15.933005344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41640"
level=info ts=2026-05-09T14:49:17.002381387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:49:17.002408089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:49:17.002417958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:49:17.00242696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:49:22.002007888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:49:23.12917636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:49:26.577967772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:49:27.001929236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:49:32.002617227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:49:32.70213558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53162"
level=warn ts=2026-05-09T14:49:37.002545494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:49:42.002616495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:49:44.495117277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:46394"
level=debug ts=2026-05-09T14:49:45.194936977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T14:49:46.984593908Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:49:46.984989943Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:49:47.002370025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:49:47.00238658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:49:47.002395399Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:49:47.00241229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:49:49.929013918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:49:49.935965735Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:49:52.00223667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:49:53.13103476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:49:54.128045145Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:49:57.00249215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:50:02.002283664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:50:02.704245511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42098"
level=warn ts=2026-05-09T14:50:07.002637455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:50:12.001684545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:50:14.498378061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:55890"
level=warn ts=2026-05-09T14:50:17.002038307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:50:17.002058078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:50:17.002080519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:50:17.002087985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:50:22.001904859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:50:23.133233727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=warn ts=2026-05-09T14:50:27.002653871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:50:29.197050106Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:50:32.002231256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:50:36.755326934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T14:50:37.002696665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:50:37.239563052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:50:37.245645977Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:50:42.001675118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:50:44.501613964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:37640"
level=info ts=2026-05-09T14:50:46.984746756Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:50:46.984755332Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T14:50:47.002055738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:50:47.002067992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:50:47.00208653Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:50:47.002093807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T14:50:47.002087235Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T14:50:47.002199521Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-09T14:50:47.002207207Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T14:50:47.002220099Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=8.715µs success=true
level=debug ts=2026-05-09T14:50:47.002488712Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-05-09T14:50:51.357893631Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T14:50:52.001964183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:50:53.134179304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:50:55.712972946Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-09T14:50:57.002339732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:00.067125184Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T14:51:02.002083876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:04.393216494Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:51:04.422342047Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T14:51:07.002323859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:08.777225937Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-09T14:51:12.001962021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:13.132287954Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
ts=2026-05-09T14:51:15.943060941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45904"
level=info ts=2026-05-09T14:51:17.001742401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:51:17.001776155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:51:17.001786853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:51:17.00185717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:17.487198513Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-09T14:51:21.841581937Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-09T14:51:22.002405213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:51:23.136704284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:51:26.19625049Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=warn ts=2026-05-09T14:51:27.001990279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:51:28.828998551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:51:28.83534481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T14:51:30.551778729Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-09T14:51:30.55180276Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-09T14:51:32.002015684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:34.905658755Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T14:51:37.00244748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:39.261214783Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T14:51:39.433825577Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:51:42.002319863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:43.615737576Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-05-09T14:51:44.747560291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T14:51:46.984674045Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:51:46.984675762Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:51:47.001999645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:51:47.002017209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:51:47.002024692Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:51:47.002016405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:47.970704967Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-09T14:51:52.002015206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:51:52.325929553Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
ts=2026-05-09T14:51:53.139603912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:51:56.680029156Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-09T14:51:57.001664926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:01.03509415Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-09T14:52:02.002182981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:05.390003034Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-09T14:52:05.810184392Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:52:07.002564398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:09.744634609Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-09T14:52:12.002683297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:14.100231198Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-09T14:52:14.510334139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43164"
level=info ts=2026-05-09T14:52:15.473737948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:52:15.481850006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T14:52:15.946009383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:49122"
level=warn ts=2026-05-09T14:52:17.001862235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:52:17.001863435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:52:17.001890247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:52:17.001901147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:52:18.455505384Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=debug ts=2026-05-09T14:52:18.455537172Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-09T14:52:22.002054987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:22.809467447Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-09T14:52:23.141549732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=warn ts=2026-05-09T14:52:27.00200585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:27.164380174Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-09T14:52:31.518567266Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-09T14:52:32.002460223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:35.874194802Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-09T14:52:37.002540235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:40.228617928Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-09T14:52:41.069434103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:52:42.001667528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:42.76562654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T14:52:42.767412595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T14:52:44.514097214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33838"
level=debug ts=2026-05-09T14:52:44.583892324Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-05-09T14:52:46.984519413Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:52:46.984868281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:52:47.002314672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:52:47.002323115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:47.002335997Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:52:47.002343815Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:52:48.938486027Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-09T14:52:52.002312292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:52:53.143474011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:52:53.293774718Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=warn ts=2026-05-09T14:52:57.001650629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:52:57.64853301Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T14:53:02.001977587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:02.003084649Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-09T14:53:06.357827088Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-09T14:53:07.00248962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:10.712522773Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-09T14:53:12.001662625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:13.389080635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T14:53:13.389137946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:53:13.3999621Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T14:53:14.839467963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:53:14.846071358Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:53:14.846094101Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T14:53:14.846116897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:53:14.852022841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T14:53:15.067839686Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-09T14:53:15.94927883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:33228"
level=warn ts=2026-05-09T14:53:17.002545922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:53:17.002547793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:53:17.002571885Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:53:17.002579509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:53:19.421897167Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T14:53:22.002488358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:53:23.146390909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:53:23.777472706Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-09T14:53:26.027418207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T14:53:27.002154267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:53:27.922654847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:53:27.928737649Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T14:53:28.132432801Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T14:53:28.132456693Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=16
level=info ts=2026-05-09T14:53:31.302046886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:53:31.30898354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:53:32.002682212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:32.48671359Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=debug ts=2026-05-09T14:53:36.84144114Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T14:53:37.001618367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:41.196497182Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-09T14:53:42.002383336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:45.3857769Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:53:45.551092692Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-09T14:53:45.95093143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:33154"
level=info ts=2026-05-09T14:53:46.984598343Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:53:46.986408034Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:53:47.001633779Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:53:47.001654465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T14:53:47.001650501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:53:47.001663335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:53:49.905728842Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-09T14:53:51.79176264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:53:51.798627352Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T14:53:52.002012449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:53:53.148954942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:53:54.261321361Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T14:53:57.001785496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:53:58.616186267Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T14:53:59.869500883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:53:59.876198754Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T14:54:01.977992469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T14:54:02.00177905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:02.970661253Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T14:54:07.002126752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:07.32503488Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=debug ts=2026-05-09T14:54:11.67998394Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-09T14:54:12.002241767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:54:14.525147801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:51282"
ts=2026-05-09T14:54:15.9517777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:56620"
level=debug ts=2026-05-09T14:54:16.034832727Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-09T14:54:16.808075582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T14:54:17.001668798Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:54:17.001692713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:17.00170779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:54:17.001718483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:54:20.3939344Z caller=checkpoint.go:372 msg="writing series" size="7.1 MB"
level=debug ts=2026-05-09T14:54:20.393956632Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.7 MB" series=12
level=warn ts=2026-05-09T14:54:22.002000678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:54:23.15086523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:54:24.745392566Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-09T14:54:27.001995204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:29.099906076Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-09T14:54:32.001645119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:33.4547305Z caller=checkpoint.go:372 msg="writing series" size="422 kB"
level=warn ts=2026-05-09T14:54:37.002636335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:37.809200344Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-09T14:54:42.002060586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:42.164549164Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
ts=2026-05-09T14:54:44.529977453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35298"
level=debug ts=2026-05-09T14:54:44.59045223Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T14:54:45.195064705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:54:45.203441061Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T14:54:45.431029067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="61 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:54:45.436802003Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T14:54:45.953529451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52820"
level=debug ts=2026-05-09T14:54:46.51875882Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-05-09T14:54:46.984873291Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:54:46.985302814Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T14:54:47.001614055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:54:47.001618188Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:54:47.001635679Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:54:47.001643926Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:54:50.873504188Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=debug ts=2026-05-09T14:54:50.873526615Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-05-09T14:54:51.435319398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T14:54:52.002053672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:54:53.153363024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:54:55.228429969Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-09T14:54:57.002056497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:54:59.583770522Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T14:55:02.001632592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:55:02.731055329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51330"
level=debug ts=2026-05-09T14:55:03.93870039Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T14:55:07.002431582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:55:08.293217072Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-09T14:55:10.279170415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:55:12.002274436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:55:12.651043572Z caller=checkpoint.go:372 msg="writing series" size="5.6 MB"
level=debug ts=2026-05-09T14:55:12.651070617Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=5
ts=2026-05-09T14:55:14.533630662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57542"
level=info ts=2026-05-09T14:55:17.001965677Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:55:17.002001321Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:55:17.002010421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:55:17.002030376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:55:17.00530242Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-09T14:55:17.009703576Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007365547s
level=warn ts=2026-05-09T14:55:22.002543613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:55:23.155495603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=warn ts=2026-05-09T14:55:27.001972964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T14:55:32.002046192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:55:33.181739981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="86 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:55:33.187836145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T14:55:33.580593975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T14:55:37.001738599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:55:40.212062758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:55:40.218750671Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T14:55:40.267916809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:55:42.0019733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:55:46.984594298Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:55:46.985209641Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:55:47.002568471Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T14:55:47.002607644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:55:47.002616187Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:55:47.002638033Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:55:47.002648561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T14:55:47.002706974Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-09T14:55:47.00271844Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T14:55:47.002742193Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.29µs success=true
level=debug ts=2026-05-09T14:55:47.003134411Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=debug ts=2026-05-09T14:55:50.421778845Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T14:55:52.002154111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:55:53.158233461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:55:53.839531708Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-09T14:55:57.002164936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:55:57.25716557Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-09T14:56:00.674473602Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T14:56:02.001620881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:04.092527495Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-09T14:56:07.002359758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:07.509735326Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-09T14:56:07.565884174Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:56:10.927553711Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=debug ts=2026-05-09T14:56:10.927574811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=8
level=warn ts=2026-05-09T14:56:12.002440235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:14.344773402Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-09T14:56:15.622588731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="86 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:56:15.628949122Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T14:56:17.002560879Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:56:17.002585958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:56:17.002594046Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:56:17.002609436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:17.762806562Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-09T14:56:21.180404795Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-09T14:56:22.002384792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:56:23.159934661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T14:56:23.786773321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:56:23.82860901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T14:56:24.598791045Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=warn ts=2026-05-09T14:56:27.002567791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:28.016519949Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=debug ts=2026-05-09T14:56:31.434406285Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=warn ts=2026-05-09T14:56:32.001852439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:33.040607896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T14:56:33.042111456Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T14:56:34.851389049Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T14:56:37.00245625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:38.26956465Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T14:56:39.546718646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:56:41.687269018Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T14:56:42.002412225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:45.105318759Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-09T14:56:46.984526713Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:56:46.986748538Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:56:47.001962734Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:56:47.001966653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:47.001987957Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:56:47.002000153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:56:48.522155822Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-09T14:56:51.940325084Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-09T14:56:52.002083022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:56:53.162685792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:56:55.357762361Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-09T14:56:55.357792222Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=warn ts=2026-05-09T14:56:57.002620096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:56:58.775827498Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-09T14:57:02.001636273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:02.193258896Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-09T14:57:05.611337976Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-09T14:57:07.002064286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:09.02931279Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-09T14:57:09.119889657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T14:57:09.785751891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:57:09.792256813Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T14:57:10.886640366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:57:10.893046008Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T14:57:11.706473569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:57:12.00167555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:12.447026839Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-09T14:57:15.86470945Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-09T14:57:15.965164382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47038"
level=warn ts=2026-05-09T14:57:17.001626176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:57:17.001658121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:57:17.001667915Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:57:17.00167527Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:57:19.282129512Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-05-09T14:57:22.002170021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:22.699751698Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-09T14:57:23.165353924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:57:26.117523231Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-09T14:57:27.002334613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:29.535152708Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-09T14:57:32.001879099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:32.953179504Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-05-09T14:57:36.371073306Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T14:57:37.00174459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:37.073024508Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:57:39.788920722Z caller=checkpoint.go:372 msg="writing series" size="424 kB"
level=debug ts=2026-05-09T14:57:39.788951182Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=13
level=warn ts=2026-05-09T14:57:42.002446463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:43.206133918Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-09T14:57:46.633064192Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-09T14:57:46.984544674Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:57:46.984960099Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:57:47.002462508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:57:47.00248306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:57:47.002494068Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:57:47.002484399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:57:48.085722232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T14:57:50.04419692Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=warn ts=2026-05-09T14:57:52.001842237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:57:53.167588807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:57:53.459456677Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-09T14:57:56.877131056Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
level=info ts=2026-05-09T14:57:56.935548721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:57:56.946195781Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:57:57.00192215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:00.29424442Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=warn ts=2026-05-09T14:58:02.002435794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:58:02.286164169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:58:02.29209139Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T14:58:03.712015487Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-09T14:58:06.777760528Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:58:07.001954216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:07.130032753Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T14:58:10.548975406Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-09T14:58:12.001891886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:13.965857203Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
ts=2026-05-09T14:58:15.969574379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45186"
level=info ts=2026-05-09T14:58:17.002189007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:58:17.002197581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:17.002219603Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:58:17.002232858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:58:17.3827536Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-05-09T14:58:20.801261744Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-09T14:58:22.00165394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:58:23.169368223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T14:58:24.218795674Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T14:58:27.001966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:27.63671601Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T14:58:31.054547975Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T14:58:32.002239067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:32.916579669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T14:58:34.473367541Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T14:58:34.473391381Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=16
level=debug ts=2026-05-09T14:58:35.196332789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T14:58:37.002523982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:37.889361574Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-09T14:58:41.306974902Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=warn ts=2026-05-09T14:58:42.0018485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:44.725421584Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-09T14:58:46.985477025Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:58:46.986150593Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:58:47.002562116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:58:47.002582627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:58:47.002591876Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:58:47.002596947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:58:48.14292862Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-09T14:58:51.560965031Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T14:58:51.869860829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:58:51.877621861Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T14:58:52.002623721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T14:58:53.172477898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:58:54.977800572Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-09T14:58:57.001694894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:58:58.375076405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="109 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:58:58.381691109Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T14:58:58.395746608Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-09T14:59:01.81377054Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T14:59:02.002139147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:05.23133833Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-09T14:59:07.002610886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:07.056847418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:59:08.649351495Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T14:59:12.002537864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:12.066796426Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-09T14:59:15.484502402Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-09T14:59:17.002414128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T14:59:17.00244127Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:59:17.002449744Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T14:59:17.002469171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:18.902576496Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T14:59:22.002239852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:22.320429627Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
ts=2026-05-09T14:59:23.175848325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T14:59:25.737865686Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-09T14:59:27.002504974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:29.167992416Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-09T14:59:29.168015291Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=16
level=warn ts=2026-05-09T14:59:32.002684839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:59:32.532747885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:59:32.54028837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:59:32.540315439Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T14:59:32.540340225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:59:32.546498773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T14:59:32.572635993Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-05-09T14:59:32.896950224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:59:34.587400162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T14:59:35.991246766Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-09T14:59:37.002369231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:39.408548595Z caller=checkpoint.go:372 msg="writing series" size="456 kB"
level=warn ts=2026-05-09T14:59:42.002028987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:42.826555249Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=debug ts=2026-05-09T14:59:46.24400697Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=debug ts=2026-05-09T14:59:46.244028647Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-09T14:59:46.462464202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T14:59:46.472657831Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T14:59:46.984778478Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T14:59:46.985548394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T14:59:47.001917739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T14:59:47.001932909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:47.001944984Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T14:59:47.001966934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T14:59:49.66163373Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T14:59:52.001667243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T14:59:53.079765111Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
ts=2026-05-09T14:59:53.177893838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T14:59:56.497631729Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-09T14:59:57.002039044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T14:59:57.186917711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:59:57.194481451Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:59:57.194499368Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T14:59:57.194524005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T14:59:57.199264895Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T14:59:58.647119633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T14:59:59.914790399Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T15:00:02.00186052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:00:03.332478397Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-09T15:00:06.750024518Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-09T15:00:07.001788723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:00:09.05458187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:00:09.061100774Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:00:10.168501317Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T15:00:12.002494134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:00:13.592266669Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-09T15:00:13.592311169Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.0 MB" series=8
ts=2026-05-09T15:00:14.56696105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:41542"
level=info ts=2026-05-09T15:00:17.00237808Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:00:17.002406083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:00:17.002415748Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:00:17.002406013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:00:17.41317242Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-09T15:00:17.4192614Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.416343734s
level=warn ts=2026-05-09T15:00:22.001644978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:00:23.179723173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:00:23.58379815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:00:23.58528818Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:00:24.012542877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:00:24.01945511Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:00:24.096758234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:00:27.002501298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:00:31.713624854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:00:31.72431108Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:00:32.001967322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:00:32.756968219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47698"
level=warn ts=2026-05-09T15:00:37.001630411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:00:42.002158109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:00:45.983266749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38694"
level=info ts=2026-05-09T15:00:46.985366793Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:00:46.986074945Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T15:00:46.986136295Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975931 n=1
level=debug ts=2026-05-09T15:00:46.986193772Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975931 n=1
level=debug ts=2026-05-09T15:00:46.986323888Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:00:47.001646379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:00:47.001661808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:00:47.001659249Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T15:00:47.001670171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:00:47.00166343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:00:47.001756456Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-09T15:00:47.001764067Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:00:47.001779833Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.462µs success=true
level=debug ts=2026-05-09T15:00:47.002225923Z caller=checkpoint.go:372 msg="writing series" size="667 kB"
level=debug ts=2026-05-09T15:00:50.420329592Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-09T15:00:51.222788952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:00:52.00162295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:00:53.182853109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:00:53.838219726Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-09T15:00:57.002032607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:00:57.256144427Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-09T15:01:00.673170267Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:01:02.002690843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:03.62765095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:01:04.091546737Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=warn ts=2026-05-09T15:01:07.002589767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:07.509070422Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-09T15:01:07.509093076Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-09T15:01:10.927038467Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-09T15:01:12.002577969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:14.344648225Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T15:01:14.574420729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:44264"
level=info ts=2026-05-09T15:01:15.187656895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:01:15.19650941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T15:01:17.002384904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:01:17.002409884Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:01:17.002420917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:01:17.002434995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:17.762514229Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-09T15:01:18.159505817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:01:18.165477197Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:01:21.179829499Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-09T15:01:21.281152585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:01:22.001753927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:01:23.185236379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:01:24.59755994Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:01:27.001647448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:28.0152514Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=debug ts=2026-05-09T15:01:31.433448089Z caller=checkpoint.go:372 msg="writing series" size="621 kB"
level=debug ts=2026-05-09T15:01:31.433471286Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-09T15:01:32.002661749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:34.850883468Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T15:01:37.002018944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:38.26815857Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:01:41.686524224Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:01:42.002532075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:45.103872035Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
ts=2026-05-09T15:01:45.987885842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40202"
level=info ts=2026-05-09T15:01:46.984523927Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:01:46.985327019Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:01:47.001723179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:01:47.001741056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:01:47.001748765Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:01:47.001764529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:47.553124652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:01:48.521270289Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-09T15:01:51.939630023Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-09T15:01:52.002433665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:01:53.186899392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:01:54.480617076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:01:55.357221987Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=debug ts=2026-05-09T15:01:55.357245566Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-09T15:01:57.002461645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:01:58.775056901Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-09T15:02:02.002255866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:02.192305178Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-09T15:02:05.610258173Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-09T15:02:07.001981138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:09.028452894Z caller=checkpoint.go:372 msg="writing series" size="377 kB"
level=info ts=2026-05-09T15:02:11.747767544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:02:11.758906219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:02:12.002633283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:12.445864531Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T15:02:13.856138223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:02:13.862172921Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:02:15.862873327Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-09T15:02:17.002209259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:02:17.002248423Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:02:17.002259646Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:02:17.002263009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:18.946775163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:02:19.281242449Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-05-09T15:02:19.281267984Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-05-09T15:02:22.002656982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:22.69874267Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
ts=2026-05-09T15:02:23.188827545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:02:26.116249363Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-09T15:02:27.001640714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:29.534641694Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-09T15:02:32.002546819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:02:32.765453257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47084"
level=debug ts=2026-05-09T15:02:32.951704642Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-05-09T15:02:36.369960847Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:02:37.001898516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:38.449246554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:02:39.78728318Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=warn ts=2026-05-09T15:02:42.001968358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:43.205460676Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=debug ts=2026-05-09T15:02:44.592831445Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:02:46.622902233Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-09T15:02:46.98455765Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:02:46.985427705Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:02:47.001851363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:02:47.00187219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:02:47.001881223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:02:47.001941307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:02:50.041067384Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=warn ts=2026-05-09T15:02:52.002463811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:02:53.191046758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:02:53.458291518Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-09T15:02:53.458315142Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-05-09T15:02:56.876221648Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-09T15:02:57.001804259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:00.293742723Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=info ts=2026-05-09T15:03:00.383198376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:03:00.393109652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:03:02.001964961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:03:02.044039782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:03:02.049600773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:03:03.711470449Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T15:03:07.002399961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:07.128722941Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T15:03:10.547156423Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-09T15:03:10.772933378Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:03:12.002601909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:13.964655688Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-09T15:03:17.001975472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:03:17.002001108Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:03:17.002009674Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:03:17.002031027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:17.382194593Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=debug ts=2026-05-09T15:03:20.799883257Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-09T15:03:22.002585412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:03:23.193074493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:03:24.217929689Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:03:27.00229798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:27.635620883Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T15:03:27.87164096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:03:31.053387581Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-09T15:03:32.001661914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:34.471505998Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-09T15:03:34.471529617Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=12
level=warn ts=2026-05-09T15:03:37.002552877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:37.888998174Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-09T15:03:41.306155669Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:03:42.002270361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:03:44.59107443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43066"
level=debug ts=2026-05-09T15:03:44.662804276Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:03:44.724203551Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
ts=2026-05-09T15:03:45.996937247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40094"
level=info ts=2026-05-09T15:03:46.985314131Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:03:46.986034672Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:03:47.002443044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:03:47.002461027Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:03:47.002469331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:03:47.002506119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:48.141926713Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-09T15:03:48.926929419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:03:48.935376638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:03:51.559952313Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:03:52.002121744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:03:53.194883581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:03:54.977144423Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-09T15:03:55.749459141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:03:55.755277415Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:03:57.001640265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:03:58.394503641Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-09T15:04:01.812907952Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:04:02.001962949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:05.230066228Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-09T15:04:05.230094578Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-09T15:04:07.002311883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:08.647869239Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T15:04:12.00230392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:12.065489981Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-09T15:04:14.594060072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:55362"
level=debug ts=2026-05-09T15:04:15.270179551Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:04:15.483669948Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
ts=2026-05-09T15:04:15.998236557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58374"
level=debug ts=2026-05-09T15:04:16.527362063Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:04:16.52928001Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:04:17.002466936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:04:17.002496018Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:04:17.002514026Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:04:17.002511179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:18.901573253Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=warn ts=2026-05-09T15:04:22.002089401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:22.319452612Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
ts=2026-05-09T15:04:23.198173813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:04:25.736469568Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T15:04:27.001751372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:29.18549768Z caller=checkpoint.go:372 msg="writing series" size="25 MB"
level=debug ts=2026-05-09T15:04:29.185530347Z caller=checkpoint.go:384 msg="flushing series" totalSize="26 MB" series=7
level=warn ts=2026-05-09T15:04:32.002511978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:32.572424339Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-09T15:04:35.989854112Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-09T15:04:37.001951438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:39.408035002Z caller=checkpoint.go:372 msg="writing series" size="501 kB"
level=warn ts=2026-05-09T15:04:42.002161464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:42.825540564Z caller=checkpoint.go:372 msg="writing series" size="345 kB"
level=debug ts=2026-05-09T15:04:42.825571558Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-09T15:04:46.000044783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:57118"
level=debug ts=2026-05-09T15:04:46.243092775Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-05-09T15:04:46.98554494Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:04:46.986836272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:04:47.002258436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:04:47.002281533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:04:47.002292023Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:04:47.002369682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:04:47.241196763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:04:47.251978312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:04:48.389748829Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:04:49.660712445Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T15:04:49.85835798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="25 MB" avg_comp="2.8 MB" total_uncomp="204 MB" avg_uncomp="23 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:04:49.864700974Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:04:52.002437667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:53.079200695Z caller=checkpoint.go:372 msg="writing series" size="984 kB"
level=debug ts=2026-05-09T15:04:53.079223884Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-09T15:04:53.200431994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:04:56.496939678Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
level=warn ts=2026-05-09T15:04:57.001928214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:04:59.91342702Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T15:05:02.002138007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:05:02.775665276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46102"
level=debug ts=2026-05-09T15:05:03.333157146Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:05:06.749015033Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-09T15:05:07.002179591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:05:09.145849903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:05:10.167084159Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-09T15:05:12.002406494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:05:13.587420718Z caller=checkpoint.go:372 msg="writing series" size="6.9 MB"
level=debug ts=2026-05-09T15:05:13.587442747Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.7 MB" series=6
level=debug ts=2026-05-09T15:05:14.111995184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:05:17.002636501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:05:17.002663565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:05:17.002671237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:05:17.002662171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:05:17.014733288Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-09T15:05:17.016153541Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.014245624s
level=warn ts=2026-05-09T15:05:22.001830009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:05:23.203558658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=warn ts=2026-05-09T15:05:27.002176276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:05:32.001613279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:05:32.778097867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54280"
level=warn ts=2026-05-09T15:05:37.002225972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:05:38.360093118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:05:39.094391268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:05:39.101588357Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T15:05:40.88694876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="25 MB" avg_comp="2.8 MB" total_uncomp="204 MB" avg_uncomp="23 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:05:40.892503734Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:05:42.001668595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:05:44.604039918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49376"
level=debug ts=2026-05-09T15:05:45.030188517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:05:46.004506985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:37558"
level=debug ts=2026-05-09T15:05:46.98445523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:05:46.984514694Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T15:05:47.002126281Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T15:05:47.002139501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:05:47.002137821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:05:47.00216374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:05:47.002172179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:05:47.002279103Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-09T15:05:47.002290495Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:05:47.002313262Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.495µs success=true
level=debug ts=2026-05-09T15:05:47.003112429Z caller=checkpoint.go:372 msg="writing series" size="890 kB"
level=debug ts=2026-05-09T15:05:50.336656192Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-09T15:05:50.336681546Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-09T15:05:52.002259289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:05:53.205124312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:05:53.669538747Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-09T15:05:57.002415509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:05:57.002558501Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-09T15:06:00.336124821Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:06:02.002336729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:03.670257939Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-09T15:06:07.00238354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:07.003668243Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-05-09T15:06:09.390931205Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:06:10.336765153Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-09T15:06:12.002583952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:13.669598748Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T15:06:16.005924918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:57270"
level=info ts=2026-05-09T15:06:17.002126315Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:06:17.002141128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:17.002162454Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:06:17.002170588Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:06:17.004801828Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-09T15:06:17.004838132Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=8
level=debug ts=2026-05-09T15:06:20.336535461Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-09T15:06:21.709061724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:06:21.719554791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:06:21.719581847Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:06:21.719605706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:06:21.734058225Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:06:22.002464679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:06:23.207436062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:06:23.669407272Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-09T15:06:26.01795693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:06:26.668704566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="251 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:06:26.674197434Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:06:26.674220185Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:06:26.674254085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="251 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:06:26.678905657Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:06:27.001946588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:27.003554975Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=debug ts=2026-05-09T15:06:30.337515389Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:06:30.337546988Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-05-09T15:06:32.002430946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:33.669980528Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-09T15:06:35.071671012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:06:35.080061152Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:06:37.002516214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:37.002581295Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-09T15:06:39.216868334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="251 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:06:39.223522847Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:06:40.336744442Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:06:42.003134948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:42.838872621Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:06:43.670077793Z caller=checkpoint.go:372 msg="writing series" size="454 kB"
ts=2026-05-09T15:06:44.610587605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33470"
level=info ts=2026-05-09T15:06:46.984876342Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:06:46.985285415Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:06:47.001851663Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:06:47.001868839Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:06:47.001875912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:06:47.001871552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:47.002989989Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-09T15:06:50.335919588Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-09T15:06:52.00261553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:06:53.210101444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:06:53.669728565Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
level=debug ts=2026-05-09T15:06:53.669752874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-09T15:06:57.002211611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:06:57.003355544Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-05-09T15:06:59.497185543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:06:59.504426775Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:07:00.336109839Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:07:02.002640204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:03.670225012Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-09T15:07:07.00185103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:07.002982986Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-09T15:07:10.336870846Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-09T15:07:12.00191792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:07:12.848106475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="251 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:07:12.855345002Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:07:13.670125493Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-09T15:07:16.446556048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:07:17.002195458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:07:17.002228695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:07:17.002236953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:07:17.002257679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:17.003538558Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-05-09T15:07:17.003551325Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-05-09T15:07:20.336833089Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-09T15:07:22.001675932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:07:23.212476067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:07:23.669848969Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-09T15:07:25.098371538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:07:27.001612077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:27.002664884Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:07:30.336305722Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-09T15:07:32.001671971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:33.669888764Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-09T15:07:37.001680752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:37.002965909Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=debug ts=2026-05-09T15:07:40.336712683Z caller=checkpoint.go:372 msg="writing series" size="287 kB"
level=warn ts=2026-05-09T15:07:42.001646717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:42.613213968Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:07:43.669976415Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-09T15:07:46.985516054Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:07:46.986555782Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:07:47.002006465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:07:47.002026315Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:07:47.002033489Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:07:47.00205112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:47.003472065Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=debug ts=2026-05-09T15:07:47.003482364Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-05-09T15:07:50.335912943Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T15:07:52.002119653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:07:53.214735489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:07:53.670011084Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-09T15:07:54.482277438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:07:54.489823845Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:07:57.001668351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:07:57.002802585Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=debug ts=2026-05-09T15:08:00.336122554Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T15:08:02.002537556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:03.669486776Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-09T15:08:07.00233124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:07.003431896Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-09T15:08:08.691928939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="275 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:08:08.700839274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:08:10.336452142Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:08:12.001918362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:13.670075475Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-09T15:08:13.754101679Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:08:14.27605735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:08:14.277631549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:08:17.001769266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:08:17.001795237Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:08:17.001803957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:08:17.001831166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:17.002955051Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-05-09T15:08:20.335911501Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:08:22.002340904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:08:23.217123478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:08:23.669960674Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-09T15:08:27.002472937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:27.002669256Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-09T15:08:30.33775849Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T15:08:30.33778094Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=13
level=warn ts=2026-05-09T15:08:32.002312331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:33.669370595Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-09T15:08:37.002181567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:37.003356161Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:08:38.507745241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:08:40.336738164Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=info ts=2026-05-09T15:08:41.904194702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:08:41.91314107Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:08:42.002416393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:43.670334958Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-09T15:08:46.984563398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:08:46.984941624Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:08:47.002523471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:08:47.002540605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:08:47.00254841Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:08:47.002562997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:47.00270655Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-09T15:08:50.33588717Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:08:52.002178856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:08:53.218647534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:08:53.66930918Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-09T15:08:57.002557894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:08:57.002779616Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:08:57.002795386Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=debug ts=2026-05-09T15:09:00.336242724Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:09:02.002372997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:09:02.804216237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53686"
level=info ts=2026-05-09T15:09:03.600227308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="275 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:09:03.606650492Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:09:03.669763843Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-09T15:09:06.921691754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:09:07.002448333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:07.002563605Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-09T15:09:10.336314829Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-09T15:09:12.002622158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:12.338143646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:09:13.669289159Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T15:09:17.002462667Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:09:17.002471494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:17.002494097Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:09:17.002504223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:09:17.002942124Z caller=checkpoint.go:372 msg="writing series" size="810 kB"
level=debug ts=2026-05-09T15:09:17.002958557Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-09T15:09:20.336246377Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T15:09:22.001675694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:09:23.22022249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:09:23.669732709Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T15:09:27.002239338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:27.040217204Z caller=checkpoint.go:372 msg="writing series" size="36 MB"
level=debug ts=2026-05-09T15:09:27.040247212Z caller=checkpoint.go:384 msg="flushing series" totalSize="37 MB" series=3
level=warn ts=2026-05-09T15:09:32.002515531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:09:37.001878897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:40.627446214Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:09:40.839158834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:09:40.848422658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:09:41.576348298Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=256 at timestamp 2026-05-09T15:09:26Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:41.69569783Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=40822 at timestamp 2026-05-09T15:09:35Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:41.695757759Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-09T15:09:32Z for stream {stream=\"build-e2e-kcnw\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:41.695764265Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-09T15:09:33Z for stream {stream=\"build-e2e-kcnw\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:41.726998178Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=26997 at timestamp 2026-05-09T15:09:32Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:41.72702446Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=24422 at timestamp 2026-05-09T15:09:36Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:41.727034316Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21802 at timestamp 2026-05-09T15:09:36Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=warn ts=2026-05-09T15:09:42.001871082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-09T15:09:42.955688833Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=154 at timestamp 2026-05-09T15:09:27Z for stream {stream=\"tekton-results\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:42.955753263Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=153 at timestamp 2026-05-09T15:09:30Z for stream {stream=\"tekton-results\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:42.955772713Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=56181 at timestamp 2026-05-09T15:09:31Z for stream {stream=\"tekton-results\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:42.95579289Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22823 at timestamp 2026-05-09T15:09:36Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:42.955804304Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=23330 at timestamp 2026-05-09T15:09:36Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:42.955842298Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5003 at timestamp 2026-05-09T15:09:27Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:42.955930195Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=4469 at timestamp 2026-05-09T15:09:37Z for stream {stream=\"build-e2e-ornb\"}" org_id=kubearchive
level=debug ts=2026-05-09T15:09:46.966864245Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-09T15:09:46.984831203Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:09:46.985347326Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=error ts=2026-05-09T15:09:46.996130599Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41193 at timestamp 2026-05-09T15:09:31Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.996180649Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=40822 at timestamp 2026-05-09T15:09:35Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.996194132Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-09T15:09:32Z for stream {stream=\"build-e2e-kcnw\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.996199834Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-09T15:09:33Z for stream {stream=\"build-e2e-kcnw\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.997221433Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=26997 at timestamp 2026-05-09T15:09:32Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.997236953Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=24422 at timestamp 2026-05-09T15:09:36Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.997245543Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21802 at timestamp 2026-05-09T15:09:36Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.997284189Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-09T15:09:29Z for stream {stream=\"build-e2e-ornb\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:46.997289153Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=197 at timestamp 2026-05-09T15:09:30Z for stream {stream=\"build-e2e-ornb\"}" org_id=kubearchive
level=info ts=2026-05-09T15:09:47.001581693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:09:47.001598013Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:09:47.001605502Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:09:47.001610937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:47.002731795Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=error ts=2026-05-09T15:09:47.66940759Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=719 at timestamp 2026-05-09T15:09:34Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-09T15:09:47.669438414Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=256 at timestamp 2026-05-09T15:09:26Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=debug ts=2026-05-09T15:09:50.337716899Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=debug ts=2026-05-09T15:09:51.158970177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:09:52.00204526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:09:53.221933888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:09:53.670103679Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=debug ts=2026-05-09T15:09:53.67012611Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-09T15:09:57.002241102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:09:57.00363507Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=debug ts=2026-05-09T15:10:00.336549236Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T15:10:02.002349199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:10:03.241123396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="36 MB" avg_comp="2.8 MB" total_uncomp="297 MB" avg_uncomp="23 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:10:03.251705461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:10:03.670572749Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T15:10:03.670595904Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-05-09T15:10:07.002487395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:10:07.002784012Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-09T15:10:07.033749323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:10:10.336075673Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T15:10:12.002416782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:10:13.669720346Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
ts=2026-05-09T15:10:14.634017977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:51480"
ts=2026-05-09T15:10:16.026769232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38134"
level=info ts=2026-05-09T15:10:17.001878617Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:10:17.001907692Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:10:17.001916488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:10:17.001910581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:10:17.003125406Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:10:20.335990965Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-09T15:10:22.002416139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:10:23.224565901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:10:23.66991156Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-09T15:10:27.001716481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:10:27.009652705Z caller=checkpoint.go:372 msg="writing series" size="7.8 MB"
level=debug ts=2026-05-09T15:10:27.009673559Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.7 MB" series=7
level=info ts=2026-05-09T15:10:28.06867405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:10:28.078279993Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T15:10:30.341913766Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-09T15:10:30.343874147Z caller=checkpoint.go:670 msg="checkpoint done" time=4m43.341399931s
level=warn ts=2026-05-09T15:10:32.002302992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:10:32.81342733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49244"
level=warn ts=2026-05-09T15:10:37.001966144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:10:37.419937258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:10:42.001983357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:10:46.984704504Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:10:46.985456266Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:10:47.001874725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:10:47.001872044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:10:47.001895463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:10:47.00190561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:10:47.001893646Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T15:10:47.002012165Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-09T15:10:47.002022517Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:10:47.00203935Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.222µs success=true
level=debug ts=2026-05-09T15:10:47.003002543Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:10:47.003012545Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-09T15:10:50.335716273Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-09T15:10:50.695363935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:10:52.001942883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:10:53.226189895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:10:53.669194952Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-09T15:10:57.002619305Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T15:10:57.002662509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:10:58.954711354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="36 MB" avg_comp="2.8 MB" total_uncomp="297 MB" avg_uncomp="23 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:10:58.960418552Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:11:00.336511724Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:11:02.001997835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:11:02.822323008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37236"
level=debug ts=2026-05-09T15:11:03.669758377Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-09T15:11:07.00166494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:07.002892445Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-09T15:11:08.527772543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:11:08.534972369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:11:10.335824278Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-09T15:11:10.667502617Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:11:10.667566876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:11:10.675926485Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:11:12.002561523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:13.669536786Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T15:11:17.002353657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:11:17.002379765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:11:17.002389089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:11:17.002395624Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:11:17.004183091Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-09T15:11:17.004200615Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=9
level=debug ts=2026-05-09T15:11:20.336591505Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-09T15:11:22.001676829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:11:23.228192378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:11:23.669834603Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-09T15:11:27.001683657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:27.002961731Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-05-09T15:11:30.285023055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:11:30.293803397Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:11:30.337686135Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T15:11:30.33771622Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=4
level=warn ts=2026-05-09T15:11:32.002561255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:33.44714589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:11:33.669711573Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-09T15:11:35.436071674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:11:37.001621633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:37.002740789Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:11:40.335985816Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:11:42.002139906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:43.669867486Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T15:11:46.985149936Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:11:46.985964445Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:11:47.002408974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:11:47.002423929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:11:47.002413435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:11:47.002430616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:11:47.002425946Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-09T15:11:47.473945878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="39 MB" avg_comp="2.8 MB" total_uncomp="321 MB" avg_uncomp="23 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:11:47.479600706Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:11:50.336673332Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-09T15:11:52.002213684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:11:52.610094961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:11:52.618110871Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:11:53.23017612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:11:53.669855117Z caller=checkpoint.go:372 msg="writing series" size="600 kB"
level=debug ts=2026-05-09T15:11:53.669877816Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-09T15:11:55.232901725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:11:55.243956564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:11:57.001925688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:11:57.003158164Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=debug ts=2026-05-09T15:12:00.335709567Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:12:02.002571625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:03.441951865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:12:03.669645594Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-09T15:12:07.002428639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:07.002896543Z caller=checkpoint.go:372 msg="writing series" size="929 kB"
level=debug ts=2026-05-09T15:12:07.002911645Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-09T15:12:10.336199355Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-09T15:12:12.00194035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:13.669590022Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-09T15:12:14.646089912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57786"
level=info ts=2026-05-09T15:12:17.002611572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:12:17.002654268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:12:17.002675942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:12:17.002621275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:17.002769864Z caller=checkpoint.go:372 msg="writing series" size="495 kB"
level=debug ts=2026-05-09T15:12:18.277765768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:12:18.279669514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T15:12:20.33631914Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=warn ts=2026-05-09T15:12:22.001671112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:12:23.233487171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:12:23.669799314Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-09T15:12:27.00175841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:27.002863982Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:12:30.335861345Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-09T15:12:32.002215035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:12:32.831353872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55634"
level=debug ts=2026-05-09T15:12:33.669972699Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-09T15:12:35.22950086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="39 MB" avg_comp="2.8 MB" total_uncomp="321 MB" avg_uncomp="23 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:12:35.238012157Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:12:37.002444464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:37.002590007Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=debug ts=2026-05-09T15:12:39.258328751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:12:40.336387648Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=debug ts=2026-05-09T15:12:40.33640886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=10
level=info ts=2026-05-09T15:12:40.386350674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:12:40.39482183Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:12:40.394848322Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:12:40.394872439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:12:40.40096219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T15:12:41.435035305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:12:41.442495564Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:12:42.002124747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:43.669106957Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-09T15:12:46.041907218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38620"
level=debug ts=2026-05-09T15:12:46.984850428Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:12:46.984855628Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T15:12:47.002408062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:12:47.002414977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:47.002435451Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:12:47.002447312Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:12:47.002700948Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=debug ts=2026-05-09T15:12:50.335962671Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T15:12:52.001997707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:12:53.236223677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:12:53.669471455Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-09T15:12:56.512305728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:12:56.519976312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:12:57.00189967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:12:57.00296198Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=debug ts=2026-05-09T15:12:58.324310473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:13:00.336450561Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T15:13:02.001773381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:03.669912321Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-09T15:13:06.028923648Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:13:07.002512649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:07.003195164Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:13:07.003210706Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=debug ts=2026-05-09T15:13:10.336036899Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:13:12.002100322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:13.669442919Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-09T15:13:14.651911212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:51230"
ts=2026-05-09T15:13:16.043194649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54398"
level=info ts=2026-05-09T15:13:17.002298216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:13:17.002323767Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:13:17.002333672Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:13:17.002329281Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-09T15:13:17.002350977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:13:17.191008118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="42 MB" avg_comp="2.8 MB" total_uncomp="345 MB" avg_uncomp="23 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:13:17.197872527Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:13:17.197889969Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:13:17.197920698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="42 MB" avg_comp="2.8 MB" total_uncomp="345 MB" avg_uncomp="23 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:13:17.203314129Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:13:20.336439161Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T15:13:20.410874505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:13:20.418031501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:13:22.002226119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:13:23.237972868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:13:23.669189166Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-09T15:13:27.002640422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:27.002681316Z caller=checkpoint.go:372 msg="writing series" size="303 kB"
level=info ts=2026-05-09T15:13:28.232101435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="42 MB" avg_comp="2.8 MB" total_uncomp="345 MB" avg_uncomp="23 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:13:28.238945961Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:13:30.338210974Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-09T15:13:30.338251278Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=7
level=warn ts=2026-05-09T15:13:32.002717116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:13:32.836139191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51382"
level=debug ts=2026-05-09T15:13:33.669423761Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T15:13:35.269051193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:13:35.275737077Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:13:36.354013052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:13:36.873676579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:13:37.002350488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:37.002445959Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:13:40.335975507Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:13:42.002231089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:43.669427923Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-09T15:13:46.985215208Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:13:46.985957107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:13:47.002385779Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:13:47.002398705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:47.002408161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:13:47.002421568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:13:47.002511489Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=debug ts=2026-05-09T15:13:50.336443506Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:13:52.002188259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:13:53.2396886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:13:53.669545758Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=debug ts=2026-05-09T15:13:53.669569707Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-09T15:13:57.002306659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:13:57.002738811Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-09T15:13:58.173600468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="42 MB" avg_comp="2.8 MB" total_uncomp="345 MB" avg_uncomp="23 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:13:58.180661974Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:14:00.336585834Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:14:02.002592837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:03.669849646Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:14:05.001738621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:14:05.00847099Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:14:07.001886832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:07.002981689Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T15:14:10.121938269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:14:10.336646303Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-09T15:14:11.734087174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:14:12.002186959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:13.669751051Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T15:14:17.001878823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:14:17.001887153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:17.001905262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:14:17.001913159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:14:17.003622662Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:14:17.003634835Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=debug ts=2026-05-09T15:14:20.336155043Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T15:14:22.001661238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:14:23.241242205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T15:14:23.406342434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:14:23.415122003Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:14:23.669923876Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T15:14:27.001718667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:27.048248502Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-09T15:14:27.048277703Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=3
level=debug ts=2026-05-09T15:14:30.336496502Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-09T15:14:32.002199866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:33.66943393Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-09T15:14:37.002631178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:37.003234837Z caller=checkpoint.go:372 msg="writing series" size="710 kB"
level=info ts=2026-05-09T15:14:39.18609505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="42 MB" avg_comp="2.8 MB" total_uncomp="345 MB" avg_uncomp="23 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:14:39.193423026Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:14:40.336964012Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=debug ts=2026-05-09T15:14:40.336985843Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-09T15:14:42.002555362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:43.245875647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:14:43.669025092Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
ts=2026-05-09T15:14:44.661499848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:34608"
level=info ts=2026-05-09T15:14:46.984623256Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:14:46.985088472Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:14:47.0026245Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:14:47.002645982Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-09T15:14:47.002640909Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T15:14:47.002654673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:14:47.002650659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:50.337925934Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-09T15:14:50.337952191Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=info ts=2026-05-09T15:14:50.659484096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:14:50.668003079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:14:52.002349994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:14:53.243586939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:14:53.670065269Z caller=checkpoint.go:372 msg="writing series" size="738 kB"
level=warn ts=2026-05-09T15:14:57.002064439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:14:57.002251415Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T15:14:59.791617395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:15:00.335685443Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T15:15:02.002178082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:15:02.843545799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54808"
level=debug ts=2026-05-09T15:15:03.669541459Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:15:03.669567951Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-09T15:15:03.928110723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:15:03.939853961Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:15:07.002385306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:15:07.002438354Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=debug ts=2026-05-09T15:15:10.335779807Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-09T15:15:11.823098798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:15:11.823160722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:15:11.830128187Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:15:12.002654132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:15:13.676531621Z caller=checkpoint.go:372 msg="writing series" size="8.3 MB"
level=debug ts=2026-05-09T15:15:13.676558214Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.3 MB" series=3
ts=2026-05-09T15:15:14.665163404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:56766"
ts=2026-05-09T15:15:16.052670261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:42928"
level=info ts=2026-05-09T15:15:17.002317847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:15:17.002355776Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:15:17.002366232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:15:17.002414615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:15:17.019688208Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-09T15:15:17.021615742Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.019444759s
level=info ts=2026-05-09T15:15:20.737763445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="45 MB" avg_comp="2.8 MB" total_uncomp="368 MB" avg_uncomp="23 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:15:20.743901513Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:15:22.002051902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:15:23.152158603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:15:23.160245614Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T15:15:23.246077784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=warn ts=2026-05-09T15:15:27.002600972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:15:32.002487076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:15:37.001660453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:15:37.51827288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:15:37.521531702Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T15:15:42.001842155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:15:43.552084696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:15:43.552122927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:15:43.561005253Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:15:46.984544589Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:15:46.985392951Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-09T15:15:46.986643292Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T15:15:46.986691356Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975932 n=1
level=debug ts=2026-05-09T15:15:46.98673053Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975932 n=1
level=info ts=2026-05-09T15:15:47.001789768Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T15:15:47.001845379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:15:47.001864136Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:15:47.001871981Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:15:47.001887953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:15:47.001904013Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-09T15:15:47.001912103Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:15:47.001927408Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.938µs success=true
level=debug ts=2026-05-09T15:15:47.002831331Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T15:15:47.002840492Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-05-09T15:15:49.444614047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:15:49.452223684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:15:50.336728336Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T15:15:52.001969766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:15:53.151268748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:15:53.159844502Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T15:15:53.248849722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T15:15:53.610376141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:15:53.618047782Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:15:53.669217652Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-09T15:15:57.002570758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:15:57.00271103Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T15:15:59.623645915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:15:59.636509094Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:16:00.335849457Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:16:02.002467398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:16:02.8472716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53688"
level=debug ts=2026-05-09T15:16:03.669087217Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-09T15:16:05.467948331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:16:05.47575533Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:16:07.002210838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:07.002383805Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=debug ts=2026-05-09T15:16:10.33563964Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-09T15:16:12.001767228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:13.669826456Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-09T15:16:16.415115064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:16:16.425487843Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:16:17.001887236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:16:17.001920229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:16:17.001929818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:16:17.001920092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:17.009188808Z caller=checkpoint.go:372 msg="writing series" size="6.4 MB"
level=debug ts=2026-05-09T15:16:17.009206524Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.3 MB" series=9
level=debug ts=2026-05-09T15:16:17.455064101Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:16:20.335528817Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:16:22.001925164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:16:23.251584374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:16:23.669843591Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-09T15:16:26.517928383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:16:27.001784775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:27.003133866Z caller=checkpoint.go:372 msg="writing series" size="418 kB"
level=debug ts=2026-05-09T15:16:30.339743497Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-09T15:16:30.339769616Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=4
level=info ts=2026-05-09T15:16:30.587070019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:16:30.595877235Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:16:32.002103764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:33.669076298Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T15:16:37.002638297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:37.002668819Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:16:40.335490716Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-09T15:16:40.412825603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:16:40.42137075Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:16:40.94388448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:16:40.952261145Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:16:42.002013726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:42.267033049Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:16:43.669911314Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
ts=2026-05-09T15:16:44.67491475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35186"
ts=2026-05-09T15:16:46.058133551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54718"
level=info ts=2026-05-09T15:16:46.306401886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:16:46.312491408Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:16:46.984554265Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:16:46.984570838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:16:47.001998529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:16:47.002006595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:16:47.002026285Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:16:47.002036239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:16:47.00220007Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-09T15:16:50.33616645Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-09T15:16:52.001885555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:16:53.253211172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:16:53.669446247Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=debug ts=2026-05-09T15:16:53.669476882Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-09T15:16:57.001856492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:16:57.003073424Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-05-09T15:17:00.335989072Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:17:02.00203953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:17:02.853102527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41774"
level=debug ts=2026-05-09T15:17:03.669635753Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=debug ts=2026-05-09T15:17:06.656863714Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:17:07.001677181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:07.003881494Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:17:07.003896877Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-09T15:17:08.414094472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:17:08.425182949Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:17:10.336733582Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-09T15:17:11.196116966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:17:12.002713975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:13.669736306Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-09T15:17:16.05947374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47892"
level=info ts=2026-05-09T15:17:17.001585925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:17:17.001618298Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:17:17.001628099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:17:17.001621809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:17.003747654Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=debug ts=2026-05-09T15:17:20.335657223Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-09T15:17:21.133918973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:17:21.152660866Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T15:17:21.601705532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:17:21.609305649Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:17:22.002751271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:17:23.255128136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:17:23.669402069Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-09T15:17:27.001650447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:27.00266865Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:17:30.335960369Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-09T15:17:32.001672141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:17:32.855111503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56860"
level=debug ts=2026-05-09T15:17:33.668924509Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-09T15:17:37.001766872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:37.003462918Z caller=checkpoint.go:372 msg="writing series" size="668 kB"
level=debug ts=2026-05-09T15:17:37.003479469Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-05-09T15:17:38.806241666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:17:38.813946622Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:17:38.813969852Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:17:38.813995373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:17:38.821039217Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:17:40.339528995Z caller=checkpoint.go:372 msg="writing series" size="785 kB"
level=debug ts=2026-05-09T15:17:40.412738033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:17:42.00221375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:43.670008307Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-09T15:17:44.951933441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:17:44.966796598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:17:46.98486413Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:17:46.985364028Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:17:47.001823602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:17:47.001821323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:17:47.001848042Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:17:47.001858113Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:17:47.003100035Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=debug ts=2026-05-09T15:17:47.003111315Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-09T15:17:50.335974243Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-09T15:17:51.215081294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:17:52.00195575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:17:53.257504811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:17:53.669594232Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T15:17:54.811621106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:17:54.818363771Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:17:57.002439448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:17:57.002482078Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=debug ts=2026-05-09T15:18:00.337450376Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T15:18:02.00221248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:18:02.901672538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:18:02.908978455Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:18:03.669250194Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-09T15:18:07.001900269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:18:07.003704304Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:18:07.00372017Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-09T15:18:09.535012453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:18:10.339038485Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:18:12.002068456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:18:12.956062754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:18:12.963138499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:18:13.669654976Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-09T15:18:15.363037462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:18:15.373645309Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:18:17.001918602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:18:17.001916Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:18:17.001944722Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:18:17.001954922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:18:17.003086771Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:18:20.335728648Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:18:22.002238906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:18:22.250196845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:18:22.258495757Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:18:23.259053288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:18:23.668835761Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-09T15:18:27.001786471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:18:27.003471123Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-05-09T15:18:30.338325949Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-09T15:18:30.338374979Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=7
level=info ts=2026-05-09T15:18:31.443956838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:18:31.449976929Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:18:32.001684712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:18:32.860435974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34228"
level=debug ts=2026-05-09T15:18:33.669758429Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-09T15:18:34.102242902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:18:35.926477482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:18:37.002521626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:18:37.00260317Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:18:40.336293758Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:18:42.001794796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:18:43.669635708Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=info ts=2026-05-09T15:18:46.985340378Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:18:46.986344747Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:18:47.001688069Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:18:47.001705024Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:18:47.001712974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:18:47.001710127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:18:47.002947431Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-09T15:18:50.336126205Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:18:52.002313849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:18:53.260553878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:18:53.669559389Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-09T15:18:57.001911712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:18:57.003503579Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:18:57.003519579Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-09T15:18:57.891111745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:18:57.901876241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:18:57.901900445Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:18:57.901923116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:18:57.908702848Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:19:00.099755401Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:19:00.099832919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="344 kB" avg_comp="344 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:19:00.103518432Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:19:00.335898045Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:19:02.002451371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:19:02.6133746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:19:02.621852577Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:19:03.669148536Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-09T15:19:07.001684807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:19:07.00283141Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-09T15:19:09.70978843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:19:09.718243737Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:19:09.817640239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:19:09.827135139Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:19:10.336594369Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-09T15:19:12.002300832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:19:13.669352792Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-05-09T15:19:13.914094097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:19:13.92231417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:19:15.266157516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:19:15.271880966Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:19:15.271903334Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:19:15.271942387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:19:15.277048963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:19:17.002437289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:19:17.002469604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:19:17.002477612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:19:17.002493391Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:19:17.002871827Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:19:17.002883787Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T15:19:18.581368131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:19:18.585270902Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:19:20.33639317Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T15:19:22.00191201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:19:23.262384895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:19:23.669672448Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:19:26.5025235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:19:26.508642252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:19:26.646151814Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:19:27.001901176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:19:27.06738674Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-09T15:19:27.067415074Z caller=checkpoint.go:384 msg="flushing series" totalSize="52 MB" series=3
level=debug ts=2026-05-09T15:19:30.038650576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:19:30.040850078Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T15:19:30.33573399Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-09T15:19:32.001657801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:19:32.865364054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33870"
level=debug ts=2026-05-09T15:19:33.66954135Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-09T15:19:37.002058598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:19:37.002959349Z caller=checkpoint.go:372 msg="writing series" size="965 kB"
level=debug ts=2026-05-09T15:19:37.002974938Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-09T15:19:40.336600478Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=warn ts=2026-05-09T15:19:42.0026202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:19:43.283341409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:19:43.292917499Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:19:43.669102657Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=info ts=2026-05-09T15:19:46.984824362Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:19:46.984834352Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:19:47.00220574Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:19:47.002232667Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:19:47.002240596Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:19:47.002273136Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T15:19:47.002352697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:19:48.09638177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:19:48.104290726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:19:50.339976251Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-09T15:19:50.340004565Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
level=debug ts=2026-05-09T15:19:51.562364317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:19:52.002409874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:19:52.633936623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:19:52.640470428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T15:19:53.264085043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:19:53.669969885Z caller=checkpoint.go:372 msg="writing series" size="776 kB"
level=info ts=2026-05-09T15:19:56.491768686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:19:56.496584957Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:19:57.00195892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:19:57.003112829Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T15:20:00.336419335Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T15:20:02.002550225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:20:02.867323375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59906"
level=debug ts=2026-05-09T15:20:03.669512208Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:20:03.669537013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-09T15:20:05.399559849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:20:07.002481563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:20:07.0025086Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-05-09T15:20:08.871309274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:20:08.879531411Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:20:09.581514301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:20:09.588859936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:20:10.335990276Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-09T15:20:12.001922496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:20:13.680776561Z caller=checkpoint.go:372 msg="writing series" size="8.8 MB"
level=debug ts=2026-05-09T15:20:13.680800691Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.8 MB" series=3
level=debug ts=2026-05-09T15:20:16.744640826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:20:17.00243173Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:20:17.002465923Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:20:17.002481328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:20:17.002534486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:20:17.004417647Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-09T15:20:17.006732191Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004683579s
level=warn ts=2026-05-09T15:20:22.002510059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:20:23.265504238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=warn ts=2026-05-09T15:20:27.002005242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:20:29.949558895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:20:29.95865956Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:20:32.002040898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:20:37.002242388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:20:40.043437827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:20:40.048993833Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:20:42.002474371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:20:44.966303721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:20:44.972496366Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:20:46.279360316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:20:46.984619096Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:20:46.985027039Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:20:47.002486068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:20:47.00250405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:20:47.002511786Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:20:47.002512475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:20:47.002517874Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T15:20:47.002655728Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-09T15:20:47.002665555Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:20:47.002683133Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.068µs success=true
level=debug ts=2026-05-09T15:20:47.003626558Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T15:20:47.003637282Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-09T15:20:50.337543255Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-09T15:20:51.044554995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:20:51.050265064Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:20:52.001876803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:20:52.360699966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:20:52.368290397Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T15:20:53.267183327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:20:53.670372012Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-09T15:20:57.002457655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:20:57.003561795Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-09T15:21:00.33677861Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:21:02.002056406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:21:02.794701686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:21:02.803115791Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:21:02.871386019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39532"
level=debug ts=2026-05-09T15:21:03.670640921Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-09T15:21:04.730991281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:21:07.001801034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:07.00404311Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-09T15:21:10.336829122Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-09T15:21:12.00214321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:13.670322985Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-09T15:21:16.270041044Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:21:16.270087368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="415 kB" avg_comp="415 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:21:16.276647336Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:21:17.002635761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:21:17.002662385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:21:17.002670225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:21:17.002664057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:17.01522141Z caller=checkpoint.go:372 msg="writing series" size="9.8 MB"
level=debug ts=2026-05-09T15:21:17.015243486Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=9
level=debug ts=2026-05-09T15:21:20.336785899Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-09T15:21:20.344871771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:21:20.352776071Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:21:22.002072697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:21:23.269137116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:21:23.669965719Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:21:27.002420584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:27.003668214Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=debug ts=2026-05-09T15:21:30.342978009Z caller=checkpoint.go:372 msg="writing series" size="7.6 MB"
level=debug ts=2026-05-09T15:21:30.343001478Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.2 MB" series=4
level=warn ts=2026-05-09T15:21:32.002039511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:21:32.2359287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:21:32.241265079Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T15:21:32.873910692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34526"
level=info ts=2026-05-09T15:21:33.649107708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:21:33.654430614Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:21:33.670488507Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-09T15:21:36.282858376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:21:36.286603914Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:21:37.001771484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:37.003955282Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-09T15:21:38.103386197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:21:38.110584201Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:21:38.110606997Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:21:38.110641138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:21:38.116549083Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:21:39.440890994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:21:39.447131458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:21:40.336591244Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:21:42.002495484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:42.576826794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:21:43.670186956Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
ts=2026-05-09T15:21:44.708858893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:50798"
ts=2026-05-09T15:21:46.085150442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41146"
level=info ts=2026-05-09T15:21:46.984550739Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:21:46.985403269Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:21:47.001843793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:21:47.001864048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:21:47.001871822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:21:47.001868515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:47.002986559Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T15:21:50.337162227Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-09T15:21:50.624017847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:21:50.633465797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:21:52.002203382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:21:53.270970273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:21:53.670001322Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-09T15:21:53.670026714Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=warn ts=2026-05-09T15:21:57.00202167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:21:57.003241075Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=info ts=2026-05-09T15:21:58.319726342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:21:58.328462916Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:22:00.337078456Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T15:22:00.992249671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:22:00.996076466Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:22:02.002248026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:22:02.875523118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40478"
level=debug ts=2026-05-09T15:22:03.669760338Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-09T15:22:03.76673046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:22:07.0020462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:07.003632791Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:22:07.003647682Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=debug ts=2026-05-09T15:22:10.336800184Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-05-09T15:22:10.855294253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:22:11.471046104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:22:11.478803643Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:22:11.478845627Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:22:11.47887066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:22:11.485915461Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:22:12.002023788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:13.669579914Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-09T15:22:16.086468464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:35742"
level=info ts=2026-05-09T15:22:17.002396195Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:22:17.002425349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:22:17.002435689Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:22:17.002462493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:17.003754414Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=debug ts=2026-05-09T15:22:20.33680517Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-05-09T15:22:20.582694259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:22:20.58640049Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:22:21.098759578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:22:21.106984821Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:22:22.002064049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:22:23.273160377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:22:23.670486397Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-09T15:22:24.157945203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:22:24.166153687Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:22:27.002529232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:27.003614015Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:22:30.337121673Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T15:22:30.444548744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:22:30.451483076Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:22:32.002332977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:33.669827307Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T15:22:35.743934905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:22:35.752599361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:22:37.002557689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:37.004288127Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-05-09T15:22:37.004306168Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-09T15:22:38.921627457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:22:38.929590064Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:22:40.338138142Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:22:40.338161835Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-09T15:22:40.532612371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:22:42.00249969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:43.67025313Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-09T15:22:44.715573736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53860"
level=info ts=2026-05-09T15:22:45.386218223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:22:45.389772561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:22:45.617470993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:22:46.088094102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:46276"
level=info ts=2026-05-09T15:22:46.984587632Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:22:46.984617147Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:22:47.002071655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:22:47.002098674Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:22:47.002109394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:22:47.002107368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:47.003787126Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T15:22:50.337135429Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T15:22:52.002514138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:22:53.275260537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:22:53.670697012Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T15:22:57.002195771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:22:57.003185311Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=info ts=2026-05-09T15:23:00.031596529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:23:00.043071389Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:23:00.336802086Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T15:23:02.002665743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:23:02.879376567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59902"
level=debug ts=2026-05-09T15:23:03.670523761Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-09T15:23:04.196556159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:23:04.202479316Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:23:07.002061716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:23:07.003986318Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:23:07.004001489Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=debug ts=2026-05-09T15:23:10.33677614Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:23:12.002590177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:23:12.038865246Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:23:13.670212677Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-09T15:23:17.002562107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:23:17.002593267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:23:17.002595988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:23:17.002604573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:23:17.003709179Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T15:23:17.613509631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:23:17.622731016Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:23:19.482886113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:23:19.486795766Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:23:20.337136209Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:23:22.001997985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:23:23.276954259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:23:23.670532025Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T15:23:23.814045244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:23:23.821888373Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:23:27.001648242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:23:27.003858555Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-09T15:23:28.561371939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:23:28.563088928Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T15:23:30.339495476Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-09T15:23:30.339523265Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=7
level=info ts=2026-05-09T15:23:30.513223118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:23:30.518584494Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:23:32.002082815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:23:32.881942164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46578"
level=debug ts=2026-05-09T15:23:33.669850388Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T15:23:34.95917969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:23:34.966288262Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T15:23:37.002267461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:23:37.003384908Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:23:37.702790698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:23:40.336855999Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:23:42.002527683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:23:43.670404996Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
ts=2026-05-09T15:23:44.722168634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:60974"
ts=2026-05-09T15:23:46.093215637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54152"
level=info ts=2026-05-09T15:23:46.98550603Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:23:46.98658485Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:23:47.001865257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:23:47.001871982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:23:47.001891836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:23:47.001899466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:23:47.003122453Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=debug ts=2026-05-09T15:23:50.336923384Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:23:52.002236783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:23:53.278779716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:23:53.669962453Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-09T15:23:53.669983875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T15:23:54.022052875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:23:54.035068872Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:23:57.002253112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:23:57.003560831Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-09T15:23:59.790843281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:23:59.79666404Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:24:00.336489068Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:24:02.001663347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:03.670602423Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:24:03.987832883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:24:03.993344986Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:24:07.002623172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:07.003721688Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T15:24:08.889428671Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:24:10.33672783Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-09T15:24:10.669689429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:24:10.67796135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:24:12.001866137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:13.670245683Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-05-09T15:24:17.002570587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:24:17.002601524Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:24:17.00261071Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:24:17.002691917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:17.004338217Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:24:17.004352593Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=debug ts=2026-05-09T15:24:17.014337819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:24:19.215472162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:24:19.222235886Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:24:19.222255518Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:24:19.222280156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:24:19.23096167Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:24:20.028168788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:24:20.039420415Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:24:20.337049653Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-09T15:24:21.620180057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:24:21.634312361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:24:22.002727758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:24:23.280369454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:24:23.669756984Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T15:24:27.001891622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:27.078381504Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-05-09T15:24:27.078409677Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=3
level=debug ts=2026-05-09T15:24:30.3365882Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-09T15:24:32.002027669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:33.670392134Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=info ts=2026-05-09T15:24:34.971007095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:24:34.984275457Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:24:35.558849969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:24:35.566731617Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:24:37.00202822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:37.003988065Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:24:37.004004449Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-09T15:24:37.04802691Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:24:40.344993542Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=warn ts=2026-05-09T15:24:42.002202749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:24:42.652108517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:24:42.658507008Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:24:43.670465738Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
ts=2026-05-09T15:24:44.730987022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:44754"
level=info ts=2026-05-09T15:24:45.626865524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:24:45.639198265Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:24:46.984596211Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:24:46.984615628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:24:47.002051823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:24:47.002077519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:24:47.002082738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:24:47.002098698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:24:47.003196671Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-09T15:24:50.339965848Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T15:24:50.339995981Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=warn ts=2026-05-09T15:24:52.002136707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:52.35176362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:24:53.28206695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:24:53.670609019Z caller=checkpoint.go:372 msg="writing series" size="823 kB"
level=info ts=2026-05-09T15:24:55.139595609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:24:55.154143422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:24:57.002605075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:24:57.00365986Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T15:25:00.339985927Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T15:25:02.00184466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:25:03.670055906Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:25:03.670080278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-09T15:25:07.002427929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:25:07.003480554Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-09T15:25:08.383606956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:25:08.393615911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T15:25:08.855731593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:25:09.019350086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:25:09.023356152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:25:10.072176545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:25:10.079767816Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:25:10.339099491Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-09T15:25:12.001760463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:25:13.679436131Z caller=checkpoint.go:372 msg="writing series" size="9.8 MB"
level=debug ts=2026-05-09T15:25:13.679458287Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.8 MB" series=3
level=info ts=2026-05-09T15:25:17.002306627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:25:17.002337645Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:25:17.002348878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:25:17.002395666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:25:17.020731553Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-09T15:25:17.022711465Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.019862201s
level=warn ts=2026-05-09T15:25:22.002575849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:25:22.370403402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:25:22.37971944Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T15:25:23.283362473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=warn ts=2026-05-09T15:25:27.001990381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:25:32.002301441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:25:32.650900881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:25:37.001996649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:25:37.682145948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:25:37.68878529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:25:40.40827286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:25:40.41557713Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:25:40.415602973Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:25:40.415640942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:25:40.420629278Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:25:42.002094479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:25:43.071563629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:25:43.200030178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:25:43.20496286Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:25:43.204984551Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:25:43.205007295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:25:43.207928312Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:25:46.98520626Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:25:46.985665472Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:25:47.001977892Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:25:47.001997763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:25:47.001992569Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T15:25:47.00200602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:25:47.002003203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:25:47.002126155Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-09T15:25:47.002136131Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:25:47.002152971Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.696µs success=true
level=debug ts=2026-05-09T15:25:47.003806844Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T15:25:47.003834896Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=debug ts=2026-05-09T15:25:50.336301601Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T15:25:52.002533019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:25:53.284868716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:25:53.669500449Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-09T15:25:54.131911789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:25:54.142894273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:25:57.001758848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:25:57.002879268Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:25:58.989257096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:25:58.996732548Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:25:59.334176866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:25:59.348928373Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:26:00.342994066Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T15:26:01.111057723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:26:01.117585684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0d33b529%3A19e0d3414bd%3A2251d941?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T15:26:01.117606344Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:26:01.117629829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:26:01.121307996Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:26:02.002071899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:26:02.895382831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45830"
level=info ts=2026-05-09T15:26:03.252584182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:26:03.256330387Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:26:03.669630694Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:26:07.002422066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:26:07.002507088Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-05-09T15:26:10.337318012Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-09T15:26:12.00185205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:26:12.057030582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:26:12.062175035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:26:13.669115765Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-09T15:26:14.390721609Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:26:17.00163577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:26:17.001628953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:26:17.001660661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:26:17.001672762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:26:17.016280405Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:26:17.016299473Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=debug ts=2026-05-09T15:26:20.341032614Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-09T15:26:20.815705921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:26:20.821541794Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:26:21.61473133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:26:21.622600588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:26:22.002536601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:26:22.934890852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:26:22.952070297Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T15:26:23.287726459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:26:23.66946087Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:26:27.001792135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:26:27.00322642Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=debug ts=2026-05-09T15:26:27.043972347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:26:30.339942802Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-09T15:26:30.339972799Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=4
level=warn ts=2026-05-09T15:26:32.002561245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:26:32.681122036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:26:32.685294613Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:26:33.66924094Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T15:26:37.00164026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:26:37.002683239Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:26:40.336234194Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:26:42.001801988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:26:43.670534037Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
ts=2026-05-09T15:26:44.746682823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:39334"
level=info ts=2026-05-09T15:26:46.985528533Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:26:46.986172863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:26:47.00254124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:26:47.002560038Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:26:47.002568406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:26:47.002598092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:26:47.002803815Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T15:26:49.119447025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:26:49.128227404Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:26:49.907432725Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:26:50.336409966Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-09T15:26:51.865800816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:26:51.869796392Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:26:52.002065293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:26:53.291469836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:26:53.670103792Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-09T15:26:53.67012596Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-05-09T15:26:57.0024015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:26:57.002499776Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-05-09T15:27:00.336050578Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T15:27:00.794857543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:27:00.800223526Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:27:02.002156197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:27:02.899577876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48874"
level=debug ts=2026-05-09T15:27:03.670079107Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-09T15:27:07.002021913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:07.003681863Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:27:07.003699531Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-09T15:27:09.467936518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:27:09.474901287Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:27:10.336253708Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-09T15:27:12.001960804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:13.669699723Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
ts=2026-05-09T15:27:14.749693669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45626"
level=info ts=2026-05-09T15:27:15.185543597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:27:15.190955903Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:27:17.002448742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:27:17.002466934Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:27:17.002488061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:27:17.002496565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:27:17.002654124Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=info ts=2026-05-09T15:27:18.663794652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:27:18.672705064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:27:20.336050395Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-09T15:27:20.3511364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:27:20.35508461Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:27:21.956106609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:27:21.957874526Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T15:27:22.002376831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:22.055633961Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T15:27:23.293961814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:27:23.669611712Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T15:27:27.002216388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:27.003305264Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:27:30.335968498Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-09T15:27:32.001655802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:27:32.901743194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58832"
level=debug ts=2026-05-09T15:27:33.669117263Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-09T15:27:35.503874503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:27:35.515700184Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:27:36.961250962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:27:36.965319133Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:27:37.002005626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:37.003464043Z caller=checkpoint.go:372 msg="writing series" size="465 kB"
level=debug ts=2026-05-09T15:27:37.003478547Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-09T15:27:40.33846963Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:27:40.33849693Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T15:27:42.002621795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:43.669162613Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:27:44.764282895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:27:44.76818948Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:27:44.768213199Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:27:44.768236068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:27:44.771341402Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T15:27:46.110698923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52598"
level=info ts=2026-05-09T15:27:46.984517389Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:27:46.985356566Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:27:47.001840222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:27:47.001914202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:27:47.001940984Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:27:47.00194921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:27:47.003242194Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-09T15:27:47.53616942Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:27:49.968475079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="56 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:27:49.977224706Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:27:50.336336821Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T15:27:52.002128698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:27:53.295649968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:27:53.669651619Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T15:27:57.002439564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:27:57.002474779Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=debug ts=2026-05-09T15:27:57.019284064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:27:59.452830596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:27:59.459197224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:27:59.459223966Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:27:59.459247937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:27:59.465450734Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:28:00.335889565Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T15:28:00.750118529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:28:00.753710638Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:28:02.002430717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:03.669980671Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T15:28:06.495960846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:28:06.50111765Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:28:06.564196794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:28:06.569247717Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:28:07.001787737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:07.00355753Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T15:28:07.003572393Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=debug ts=2026-05-09T15:28:10.336401524Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:28:12.002284966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:12.503134331Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:28:13.669464258Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
ts=2026-05-09T15:28:16.112023752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:57638"
level=info ts=2026-05-09T15:28:17.002410311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:28:17.002434638Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:28:17.002438326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:28:17.002455004Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:28:17.00248418Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T15:28:17.345620115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:28:17.351440882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:28:18.332479417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:28:18.339734046Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:28:20.336174197Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:28:22.002099302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:28:22.956616049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:28:22.965570128Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:28:23.297673319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:28:23.669748404Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T15:28:27.002530792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:27.002570892Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-09T15:28:30.33928264Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-09T15:28:30.339305565Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=7
level=info ts=2026-05-09T15:28:31.361928944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:28:31.365679666Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:28:32.001740058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:28:32.906433929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60574"
level=debug ts=2026-05-09T15:28:33.66983079Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-09T15:28:36.940721566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:28:37.002623318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:37.002732573Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:28:40.335972807Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:28:42.002633922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:43.669551901Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
ts=2026-05-09T15:28:44.760583917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:34340"
level=debug ts=2026-05-09T15:28:45.217518394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:28:46.984592445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:28:46.984607211Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:28:47.001983752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:28:47.00200198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:28:47.002021682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:28:47.002030266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:28:47.003270764Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
level=info ts=2026-05-09T15:28:48.579694946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:28:48.586949628Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:28:48.586970883Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:28:48.586994498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:28:48.593059816Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:28:50.335731779Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T15:28:50.587692862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:28:50.595057972Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:28:52.002499068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:28:53.086401035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:28:53.093585388Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T15:28:53.299631689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:28:53.669468997Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-09T15:28:53.669493675Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-09T15:28:57.002053984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:28:57.003566004Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:29:00.336486588Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:29:02.002542783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:29:02.90842596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34518"
level=info ts=2026-05-09T15:29:02.982129847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:29:02.985674664Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:29:03.669880888Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:29:06.686538474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:29:06.694533045Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:29:07.002345305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:07.002380787Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T15:29:10.33682151Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-09T15:29:12.002006753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:13.669549805Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-09T15:29:14.36650634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:29:14.376106898Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:29:16.116186773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41688"
level=info ts=2026-05-09T15:29:16.149719949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:29:16.153393672Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:29:16.413322702Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:29:17.001588308Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:29:17.001613802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:29:17.001621813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:29:17.001618314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:17.003472154Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:29:17.003485755Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=debug ts=2026-05-09T15:29:20.335953678Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T15:29:22.00195237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:29:22.090265423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:29:22.093772674Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T15:29:23.302229762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:29:23.669756951Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T15:29:27.001744081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:27.059337231Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T15:29:27.059361599Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=3
level=debug ts=2026-05-09T15:29:30.336553987Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-09T15:29:32.002004205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:29:32.824865989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:29:32.839952153Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:29:33.122760217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:29:33.128613945Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:29:33.669681978Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=warn ts=2026-05-09T15:29:37.002230838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:37.004144773Z caller=checkpoint.go:372 msg="writing series" size="896 kB"
level=debug ts=2026-05-09T15:29:37.004161515Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-09T15:29:38.359730471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:29:40.336904786Z caller=checkpoint.go:372 msg="writing series" size="732 kB"
level=warn ts=2026-05-09T15:29:42.001969573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:43.66980241Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=debug ts=2026-05-09T15:29:43.669849536Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-09T15:29:45.925775868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:29:45.933787961Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:29:46.984663536Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:29:46.984698576Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:29:47.002082813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:29:47.002105602Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:29:47.002103061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:29:47.002113802Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:29:47.003207392Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-09T15:29:49.589826041Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:29:50.33912037Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T15:29:50.339143164Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=warn ts=2026-05-09T15:29:52.001940357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:29:52.11427071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:29:52.118328368Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-09T15:29:53.304604509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:29:53.670115307Z caller=checkpoint.go:372 msg="writing series" size="978 kB"
level=warn ts=2026-05-09T15:29:57.001616011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:29:57.002730433Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T15:30:00.336184463Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T15:30:02.002381829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:30:02.30957871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:30:02.313979304Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
ts=2026-05-09T15:30:02.913141021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53460"
level=debug ts=2026-05-09T15:30:03.670201026Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:30:03.670223728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-09T15:30:04.769110213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:30:04.777573134Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:30:07.002566782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:30:07.002616Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-09T15:30:10.336743543Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-09T15:30:10.753178803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:30:10.756943715Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:30:12.00213786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:30:13.680550613Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-09T15:30:13.680576301Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=3
ts=2026-05-09T15:30:14.769984404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57040"
level=info ts=2026-05-09T15:30:15.381320958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:30:15.387262974Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:30:15.918714435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:30:17.001716193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:30:17.001707626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:30:17.001740441Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:30:17.001753594Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:30:17.023250384Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-09T15:30:17.027214414Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.02491913s
level=warn ts=2026-05-09T15:30:22.001687639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:30:23.306226879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=warn ts=2026-05-09T15:30:27.002017537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:30:28.309220941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:30:28.318340032Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:30:28.777370942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:30:29.177214595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:30:29.184192587Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:30:32.002298063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:30:32.914857291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42596"
level=info ts=2026-05-09T15:30:33.435173596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:30:33.438938179Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:30:37.001759685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:30:42.001859682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:30:45.82272416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:30:45.830397593Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:30:46.984540651Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:30:46.985663862Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-09T15:30:46.985885757Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T15:30:46.985931315Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975933 n=1
level=debug ts=2026-05-09T15:30:46.985971535Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975933 n=1
level=info ts=2026-05-09T15:30:47.00195717Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T15:30:47.001958024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:30:47.001972432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:30:47.00198789Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:30:47.001995805Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:30:47.002052184Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-09T15:30:47.002060319Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:30:47.002075168Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.565µs success=true
level=debug ts=2026-05-09T15:30:47.003032443Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T15:30:47.003041334Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=debug ts=2026-05-09T15:30:47.982802937Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:30:50.33635544Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T15:30:52.002280658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:30:53.161686701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:30:53.169996385Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:30:53.170017125Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:30:53.170041195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:30:53.177167377Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:30:53.308619105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:30:53.669571569Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-09T15:30:57.002512331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:30:57.002528589Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:30:58.304668852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:30:58.309087737Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:31:00.33666616Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:31:02.001621412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:03.669924771Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:31:03.868447153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:31:03.87763736Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:31:05.192553966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:31:05.198112086Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:31:07.001762084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:07.003145192Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-09T15:31:10.336355947Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-05-09T15:31:12.002090464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:13.417254412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:31:13.418765621Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T15:31:13.669619659Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-09T15:31:13.892443545Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:31:17.002583277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:31:17.002596079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:31:17.00261905Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:31:17.002636997Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:31:17.012212063Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:31:17.012232667Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=debug ts=2026-05-09T15:31:20.336046949Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-09T15:31:22.001991596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:31:22.34471401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:31:22.351578997Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:31:22.755283099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:31:22.763240544Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T15:31:23.31080121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:31:23.669513592Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:31:27.001864744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:27.00318933Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=info ts=2026-05-09T15:31:29.103573862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:31:29.113187754Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:31:30.341426936Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:31:30.341451095Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=warn ts=2026-05-09T15:31:32.002131897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:31:33.208762439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:31:33.212294098Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:31:33.669141664Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-09T15:31:33.875844918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:31:33.879961171Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:31:37.001747962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:37.00289361Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:31:40.335883227Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:31:42.002539413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:43.493912329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:31:43.669731494Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
ts=2026-05-09T15:31:46.127390316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38742"
level=info ts=2026-05-09T15:31:46.984828892Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:31:46.985511083Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:31:47.001743097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:31:47.001774781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:31:47.001792864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:31:47.001800533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:31:47.002903045Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-09T15:31:47.200736292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:31:47.205051996Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:31:50.336252672Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=debug ts=2026-05-09T15:31:51.827457646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:31:52.002357499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:31:53.313169444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T15:31:53.497588148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:31:53.503376164Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:31:53.503396434Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:31:53.503432222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:31:53.508488497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:31:53.670248976Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=debug ts=2026-05-09T15:31:53.670270853Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=warn ts=2026-05-09T15:31:57.002530679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:31:57.002642266Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-05-09T15:32:00.336034798Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:32:02.002235574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:32:03.669986483Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-09T15:32:07.002484845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:32:07.003587517Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:32:07.003602804Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T15:32:07.648496942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:32:07.659643562Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:32:09.068492964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:32:09.075837081Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:32:10.33619255Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-09T15:32:10.340237615Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:32:10.658142667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:32:10.664825357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:32:12.001871234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:32:13.669666859Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-09T15:32:17.002049358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:32:17.002069263Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:32:17.002090341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:32:17.002098373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:32:17.003310729Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=debug ts=2026-05-09T15:32:20.336495036Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=warn ts=2026-05-09T15:32:22.002266772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:32:23.314600485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:32:23.669131379Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-09T15:32:24.186859801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:32:24.197441532Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:32:27.001719032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:32:27.002861295Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:32:30.335780872Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T15:32:31.548329384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:32:31.552449317Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:32:31.552469402Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:32:31.552491576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:32:31.560653822Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:32:32.001853329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:32:33.327700844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:32:33.331552505Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:32:33.607316587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:32:33.611123165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:32:33.611141527Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:32:33.61116398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:32:33.614519144Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:32:33.669673447Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T15:32:37.001669964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:32:37.002991338Z caller=checkpoint.go:372 msg="writing series" size="467 kB"
level=debug ts=2026-05-09T15:32:37.00300478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-09T15:32:38.221470052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:32:40.337137371Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:32:40.337161943Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T15:32:42.002629295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:32:42.178030077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:32:42.184164786Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:32:43.669909237Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-09T15:32:44.785270058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:47410"
level=info ts=2026-05-09T15:32:45.045215489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:32:45.049320765Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
ts=2026-05-09T15:32:46.130162938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54802"
level=info ts=2026-05-09T15:32:46.984803Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:32:46.98480218Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:32:47.00208583Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:32:47.002085111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:32:47.002106453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:32:47.002117446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:32:47.003877883Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T15:32:50.092883246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:32:50.306439106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:32:50.309902516Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:32:50.335940697Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T15:32:52.002140089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:32:53.315626072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:32:53.669352984Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-09T15:32:57.002620942Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=warn ts=2026-05-09T15:32:57.00263528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:00.335776565Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T15:33:01.369792958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:33:01.378407278Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:33:01.900767715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:33:01.907700059Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:33:02.002073066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:03.669698072Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-09T15:33:07.001953493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:07.003871301Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T15:33:07.003885959Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=8
level=debug ts=2026-05-09T15:33:10.335729208Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:33:12.002001953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:12.203034938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:33:12.203081535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:33:12.209194693Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=info ts=2026-05-09T15:33:13.173767491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:33:13.177562664Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:33:13.669619493Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=info ts=2026-05-09T15:33:14.921947084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:33:14.925730335Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:33:17.002557569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:33:17.002579056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:17.002586931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:33:17.002597259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:33:17.0026651Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-09T15:33:20.336298636Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:33:22.002373712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:33:22.714450034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:33:22.721884907Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:33:22.761914857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:33:22.765659653Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T15:33:23.317199479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:33:23.669167248Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T15:33:27.002492169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:27.002621795Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-09T15:33:29.450699209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:33:29.456513606Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=debug ts=2026-05-09T15:33:30.338847039Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-09T15:33:30.33887061Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=7
level=warn ts=2026-05-09T15:33:32.00174577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:33.669279412Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T15:33:36.490757189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:33:36.495984172Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:33:37.002335048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:37.002437212Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:33:40.336568071Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:33:42.001997458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:43.670159549Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
ts=2026-05-09T15:33:44.79163039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45380"
level=debug ts=2026-05-09T15:33:45.278668845Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:33:45.278725352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:33:45.282627011Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:33:46.984557616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:33:46.984976538Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:33:47.002309224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:33:47.002327828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:33:47.002338373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:33:47.002401846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:47.002515054Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=debug ts=2026-05-09T15:33:47.002527526Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-09T15:33:49.571616339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:33:50.336022133Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:33:52.002414206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:33:53.318889725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T15:33:53.608681314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:33:53.612316588Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:33:53.669495632Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-09T15:33:57.001991837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:33:57.003438906Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-09T15:33:57.996999108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:33:58.003752467Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:33:58.084981058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:33:58.088758234Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:33:59.112193984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:33:59.11589209Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:34:00.336022592Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-09T15:34:01.885487871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:34:01.894142186Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:34:02.002700627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:34:03.148054202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:34:03.15525482Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:34:03.669103092Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-09T15:34:07.002530762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:34:07.002636332Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:34:09.192366312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:34:09.19842057Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=debug ts=2026-05-09T15:34:10.33576029Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-09T15:34:12.002256622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:34:12.793625811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:34:12.797469873Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:34:12.797488785Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:34:12.797509205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:34:12.800898079Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:34:13.670042665Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-09T15:34:13.670065373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-05-09T15:34:16.135564685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:51990"
level=info ts=2026-05-09T15:34:17.002321029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:34:17.00234843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:34:17.002356816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:34:17.00236949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:34:17.002959136Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:34:20.336546167Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-09T15:34:20.910193405Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:34:20.910236224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:34:20.91592238Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:34:21.767991834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:34:21.773910682Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:34:22.002416727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:34:23.321111119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:34:23.670137845Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:34:26.062280748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:34:26.0661328Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:34:27.002550361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:34:27.059480549Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T15:34:27.059507383Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=4
level=debug ts=2026-05-09T15:34:30.335851354Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:34:31.742671276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:34:31.747183427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:34:32.001674811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:34:33.669984802Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-09T15:34:37.002064769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:34:37.003948591Z caller=checkpoint.go:372 msg="writing series" size="936 kB"
level=debug ts=2026-05-09T15:34:37.003967495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-09T15:34:38.022881182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:34:38.027196346Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:34:40.336490843Z caller=checkpoint.go:372 msg="writing series" size="798 kB"
level=debug ts=2026-05-09T15:34:40.951438883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:34:41.332706291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:34:41.340206626Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:34:42.001766635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:34:43.291012876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:34:43.294495473Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:34:43.669715736Z caller=checkpoint.go:372 msg="writing series" size="436 kB"
level=debug ts=2026-05-09T15:34:43.669738102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
ts=2026-05-09T15:34:44.798146415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38510"
level=info ts=2026-05-09T15:34:45.354334608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:34:45.358057344Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:34:45.850569118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:34:46.984824511Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:34:46.985294067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:34:47.001595054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:34:47.001615245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:34:47.001623275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:34:47.001669129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:34:47.002691623Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T15:34:47.750635089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:34:47.758859894Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:34:47.758882968Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:34:47.75890383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:34:47.765375915Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:34:50.339634374Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T15:34:50.33978179Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-09T15:34:51.954341226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:34:51.959465924Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:34:52.002177967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:34:53.322418144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:34:53.669767615Z caller=checkpoint.go:372 msg="writing series" size="819 kB"
level=info ts=2026-05-09T15:34:55.844088183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:34:55.85023153Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=debug ts=2026-05-09T15:34:57.00231395Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T15:34:57.00234395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:35:00.335678889Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T15:35:00.41200256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:35:00.420600297Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:35:01.548991667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:35:01.553114119Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:35:02.002014399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:35:02.97950017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:35:02.989782747Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:35:03.669503982Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:35:03.669527815Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-09T15:35:07.001776773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:35:07.002985349Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T15:35:10.247782833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:35:10.254048251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:35:10.33628408Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-09T15:35:12.002658958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:35:13.683739139Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T15:35:13.683761913Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=3
level=debug ts=2026-05-09T15:35:15.281084034Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:35:17.00158168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:35:17.001608175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:35:17.001617607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:35:17.001638789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:35:17.024228814Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-09T15:35:17.027192371Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.024994367s
level=info ts=2026-05-09T15:35:19.878344859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:35:19.882350908Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:35:22.002140619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:35:23.324795695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T15:35:23.85402551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:35:23.857530155Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:35:24.290731604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:35:24.29237094Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:35:26.056367024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:35:26.063756551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:35:27.002523843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:35:29.665922132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:35:29.669547535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:35:32.001635921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:35:32.937778593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56664"
level=warn ts=2026-05-09T15:35:37.002422182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:35:37.587705469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:35:37.592271824Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:35:38.387545091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:35:38.395382464Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:35:38.395404242Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:35:38.39543204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:35:38.402206376Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:35:40.53999206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:35:42.002042697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:35:44.804195709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:51216"
level=info ts=2026-05-09T15:35:46.984531692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:35:46.984904389Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:35:47.002181339Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T15:35:47.002189674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:35:47.002189232Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:35:47.002206972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:35:47.00221529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:35:47.00232342Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-09T15:35:47.002332686Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:35:47.002351573Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.2µs success=true
level=debug ts=2026-05-09T15:35:47.004708266Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T15:35:47.004767077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-09T15:35:49.089654595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:35:49.094421363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:35:50.336787052Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T15:35:52.002302504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:35:53.031128288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:35:53.038421629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T15:35:53.326784829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:35:53.669484504Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-09T15:35:55.136653596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:35:55.141729452Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=info ts=2026-05-09T15:35:56.241880259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:35:56.24711438Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:35:57.002083245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:35:57.0031921Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:35:58.357734874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:35:58.364970635Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:36:00.336281936Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-09T15:36:01.282376753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:36:02.001858148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:36:02.940056339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56228"
level=debug ts=2026-05-09T15:36:03.669715551Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:36:07.002544616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:36:07.002647666Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-09T15:36:10.336539113Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-09T15:36:11.888008428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:36:11.895084086Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:36:12.002126027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:36:12.29979596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:36:12.303644624Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:36:12.485799316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:36:13.669512107Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T15:36:16.144075571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41830"
level=info ts=2026-05-09T15:36:17.002492062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:36:17.002516804Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:36:17.00252454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:36:17.002516524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:36:17.014357034Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:36:17.014377273Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=debug ts=2026-05-09T15:36:20.336309896Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-09T15:36:21.8836751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:36:21.890454604Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:36:22.002477773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:36:23.329130603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:36:23.669893552Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:36:27.00161436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:36:27.003145491Z caller=checkpoint.go:372 msg="writing series" size="495 kB"
level=info ts=2026-05-09T15:36:28.844150489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:36:28.848282333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:36:29.83209172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:36:29.838806714Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:36:30.347525535Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T15:36:30.347550507Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=warn ts=2026-05-09T15:36:32.001917437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:36:32.941621241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49096"
level=debug ts=2026-05-09T15:36:33.669325686Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-09T15:36:36.162713384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:36:36.166688443Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:36:37.002000606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:36:37.003122867Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:36:37.655890373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:36:40.33678756Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-09T15:36:40.928625081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:36:40.934128914Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:36:40.967175308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:36:40.974473086Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:36:42.002175141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:36:43.670435384Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=debug ts=2026-05-09T15:36:44.325611767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:36:44.61350049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:36:44.617644965Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-09T15:36:44.810049344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57406"
level=info ts=2026-05-09T15:36:46.984677646Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:36:46.985115529Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:36:47.002475147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:36:47.002502857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:36:47.002535848Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:36:47.002548414Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:36:47.00260081Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-09T15:36:47.044848504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:36:47.049547958Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=debug ts=2026-05-09T15:36:50.336725417Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-09T15:36:52.002241491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:36:53.331987353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:36:53.670621936Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=debug ts=2026-05-09T15:36:53.670689354Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-05-09T15:36:57.002040396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:36:57.003228006Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-09T15:36:57.866674863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:36:57.87265103Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:37:00.336400447Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:37:02.001659323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:37:03.669884759Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T15:37:05.680703018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:37:05.684223212Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:37:06.649139691Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:37:07.00205143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:37:07.003693953Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:37:07.003707537Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T15:37:08.024782052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:37:08.031447422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:37:10.336723405Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-09T15:37:12.002157308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:37:13.669897757Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-09T15:37:17.002011899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:37:17.002011305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:37:17.002035094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:37:17.002042998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:37:17.003382781Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=debug ts=2026-05-09T15:37:20.336588821Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-09T15:37:22.002481337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:37:22.343373752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:37:22.349008631Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
ts=2026-05-09T15:37:23.333965013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:37:23.669385486Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-09T15:37:25.479862684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:37:25.486523887Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:37:27.002442521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:37:27.002537352Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-09T15:37:30.336678395Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T15:37:31.892984764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:37:31.901646863Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:37:31.901667171Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:37:31.901692054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:37:31.908624945Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:37:32.002620272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:37:33.471672405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:37:33.475265645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:37:33.669743788Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-05-09T15:37:34.758927402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:37:34.76336613Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:37:36.628972244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:37:36.633426219Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=warn ts=2026-05-09T15:37:37.001857483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:37:37.003179369Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
level=debug ts=2026-05-09T15:37:37.003194002Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-09T15:37:39.158258787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:37:39.164168517Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:37:39.519374489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:37:40.337339917Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:37:40.337362745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T15:37:42.00178303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:37:42.190768454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:37:42.193941127Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:37:42.375358082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:37:43.669499031Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:37:46.371630241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:37:46.375373712Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:37:46.984335196Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:37:46.98538422Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-09T15:37:47.001675681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:37:47.001700209Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:37:47.001723066Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:37:47.001745352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:37:47.003169922Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=info ts=2026-05-09T15:37:49.110575489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:37:49.11820309Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:37:50.336629367Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T15:37:52.001855205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:37:53.335252184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:37:53.670214437Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-09T15:37:57.00257703Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=warn ts=2026-05-09T15:37:57.002586453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:00.336508284Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T15:38:02.002248047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:03.669545373Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-09T15:38:05.306436851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:38:05.314023164Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:38:06.654755095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:38:06.661533663Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:38:07.00256239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:07.007399195Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T15:38:07.007424059Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=8
level=debug ts=2026-05-09T15:38:08.647868785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:38:10.336500518Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:38:12.002100422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:13.669455113Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-09T15:38:13.806998704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:38:13.810763478Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:38:14.089623863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:38:14.096329584Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:38:14.819967622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53084"
level=info ts=2026-05-09T15:38:17.002177053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:38:17.002203686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:38:17.002190283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:38:17.002212356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:38:17.003335894Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T15:38:19.313396545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:38:19.31729486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:38:20.336034317Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:38:22.002177001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:38:23.336905954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:38:23.669766014Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T15:38:26.212787009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:38:26.218684382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:38:26.218704495Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:38:26.218742561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:38:26.223849202Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:38:27.001982803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:27.003082342Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-09T15:38:28.467781049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:38:28.471475344Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:38:28.471496104Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:38:28.471519774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:38:28.474257931Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:38:29.367753526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:38:29.37179002Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:38:30.338696603Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-09T15:38:30.338719962Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=7
level=warn ts=2026-05-09T15:38:32.002611136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:38:32.461416484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:38:32.466089444Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=debug ts=2026-05-09T15:38:33.397385607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:38:33.670200635Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T15:38:33.833644176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:38:33.837034366Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:38:37.002126709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:37.003287932Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-09T15:38:39.041180172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:38:39.367945794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:38:39.374939451Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:38:40.336663247Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:38:42.002664324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:43.669989177Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-09T15:38:46.984524503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:38:46.984938499Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:38:47.002302884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:38:47.002321266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:38:47.002329369Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:38:47.00234494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:47.003503966Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-09T15:38:47.406244575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:38:47.40991587Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:38:50.336306465Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T15:38:51.554640437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:38:51.561640561Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:38:52.001802457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:38:53.339106281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:38:53.66949341Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-09T15:38:53.891112296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:38:53.898089457Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:38:56.514425817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:38:56.518611306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:38:56.518632162Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:38:56.518655895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:38:56.52189047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:38:57.00224504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:38:57.00345587Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:38:57.00347451Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=debug ts=2026-05-09T15:39:00.336626992Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T15:39:02.002344683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:03.669965771Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:39:06.792644667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:39:06.800277779Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:39:07.002469625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:07.00259748Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-09T15:39:07.889095888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:39:07.890696861Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:39:08.707624174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:39:08.710881901Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:39:10.336669292Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T15:39:10.721975323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:39:10.728256497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:39:12.002313206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:13.669840591Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-09T15:39:13.787484909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:39:15.927193932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:39:15.931446382Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:39:16.154092474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:39:16.157862846Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:39:17.002309384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:39:17.002334292Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:39:17.002343151Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:39:17.002355847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:17.003776234Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:39:17.00378797Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T15:39:18.96731625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:39:18.971085091Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:39:20.336467663Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T15:39:22.002241165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:39:22.809237342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:39:22.813188882Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T15:39:23.341152537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:39:23.669790267Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:39:25.815043651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:39:25.819446912Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=warn ts=2026-05-09T15:39:27.002376992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:27.057328327Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T15:39:27.057353319Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=debug ts=2026-05-09T15:39:30.336006296Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-09T15:39:32.00213809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:39:32.883693042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:39:32.892738309Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T15:39:32.953354281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42044"
level=debug ts=2026-05-09T15:39:33.669840512Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=warn ts=2026-05-09T15:39:37.001984759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:37.006026531Z caller=checkpoint.go:372 msg="writing series" size="950 kB"
level=debug ts=2026-05-09T15:39:37.006046864Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-09T15:39:38.183087883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:39:40.337180518Z caller=checkpoint.go:372 msg="writing series" size="896 kB"
level=warn ts=2026-05-09T15:39:42.00214092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:43.669749539Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=debug ts=2026-05-09T15:39:43.669777172Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
ts=2026-05-09T15:39:44.829997367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45906"
level=info ts=2026-05-09T15:39:46.984515622Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:39:46.986592804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:39:47.001754283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:39:47.001775989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:39:47.001786375Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:39:47.001864314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:47.002947899Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-09T15:39:50.338711373Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T15:39:50.33873327Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-09T15:39:51.46146908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:39:51.481517294Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:39:51.659857449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:39:51.667389707Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:39:52.001927149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:39:53.34340942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T15:39:53.558928285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:39:53.563384956Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:39:53.670707998Z caller=checkpoint.go:372 msg="writing series" size="822 kB"
level=warn ts=2026-05-09T15:39:57.002045487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:39:57.003176181Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-09T15:39:58.038607291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:39:58.042406631Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:39:58.963359775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:40:00.29924708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:40:00.305716917Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:40:00.336864516Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T15:40:02.002374663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:40:03.670111621Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:40:03.670133486Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-09T15:40:06.614627793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:40:07.001950059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:40:07.003089502Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-09T15:40:10.336307043Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-09T15:40:12.002631172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:40:12.902942137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:40:12.908111029Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:40:13.376295196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:40:13.380335406Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T15:40:13.380355091Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:40:13.380377808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:40:13.383378761Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:40:13.64290204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:40:13.647125562Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:40:13.678082873Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-09T15:40:13.678097634Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
ts=2026-05-09T15:40:14.83362559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59284"
level=info ts=2026-05-09T15:40:17.001612854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:40:17.00164196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:40:17.001650024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:40:17.001641751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:40:17.024570519Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-09T15:40:17.026028497Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.023552569s
level=warn ts=2026-05-09T15:40:22.001754643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:40:23.345854171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T15:40:25.744766235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:40:25.75162808Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:40:25.751647987Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:40:25.751670431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:40:25.756208428Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=warn ts=2026-05-09T15:40:27.002364452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:40:27.641943448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:40:27.648326333Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:40:30.894481962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:40:30.898250943Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:40:32.001725369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:40:34.965481494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:40:36.834960703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:40:37.001900667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:40:37.141232993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:40:37.145041913Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:40:37.708552671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:40:37.715219785Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:40:41.300636365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:40:41.307726319Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:40:42.002182189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:40:44.740463541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:40:44.743800143Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:40:44.743841163Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:40:44.743861317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:40:44.746532056Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:40:44.838847098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:40:44.843043669Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=info ts=2026-05-09T15:40:45.718289068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:40:45.723254013Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:40:46.984610044Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:40:46.984639803Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:40:47.001895965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:40:47.001913698Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:40:47.001923205Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:40:47.00193494Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T15:40:47.001961776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:40:47.002060389Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-09T15:40:47.00206884Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:40:47.002084738Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.652µs success=true
level=debug ts=2026-05-09T15:40:47.003196664Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-09T15:40:47.00320619Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
level=debug ts=2026-05-09T15:40:50.336704963Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T15:40:52.002304785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:40:53.348131246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:40:53.669827552Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-09T15:40:53.760024355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:40:53.763938912Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:40:57.001785914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:40:57.002945638Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:40:58.435631808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:40:58.43959493Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:40:59.719270169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:40:59.722933475Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:41:00.071026511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:41:00.075419042Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-09T15:41:00.07543703Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:41:00.075464442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:41:00.078770612Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:41:00.33636599Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:41:02.002106607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:03.670032673Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:41:07.002112997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:07.003318176Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-09T15:41:07.115575428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:41:10.336091113Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-09T15:41:10.992133779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:41:10.997970837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:41:12.002132598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:13.66939636Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T15:41:14.840265874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49096"
level=debug ts=2026-05-09T15:41:15.514793068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:41:17.001665052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:41:17.001691812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:41:17.001700979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:41:17.001712772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:17.014943417Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:41:17.014971297Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=info ts=2026-05-09T15:41:19.246013473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:41:19.252527769Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:41:19.252548042Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:41:19.252570025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:41:19.258414977Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:41:20.336042411Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-09T15:41:22.002489393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:41:23.349259857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:41:23.669970266Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:41:23.874180442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:41:23.879177237Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=warn ts=2026-05-09T15:41:27.002482006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:27.002617942Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-09T15:41:28.020291398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:41:28.029184079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:41:28.453301209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:41:28.460052233Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:41:30.345140502Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T15:41:30.345164685Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=info ts=2026-05-09T15:41:30.52632455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:41:30.533531934Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:41:31.171235536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:41:31.178400077Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:41:32.002488586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:33.669503921Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-09T15:41:34.583556553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:41:34.587670265Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:41:36.202537939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:41:36.202553022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:41:36.207610011Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:41:37.002379124Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-09T15:41:37.00237835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:41:38.441619826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:41:38.444739742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:41:39.79055521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:41:39.79592982Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:41:40.016659756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:41:40.020569544Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:41:40.336344544Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:41:42.002105097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:43.669705309Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
ts=2026-05-09T15:41:44.843023708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33648"
level=info ts=2026-05-09T15:41:46.984521526Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:41:46.984917304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:41:47.002305799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:41:47.002307553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:47.002324021Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:41:47.002333645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:41:47.002380174Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-09T15:41:50.336321383Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-05-09T15:41:50.64216797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:41:50.647129201Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:41:52.002387187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:41:53.350728032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:41:53.670088968Z caller=checkpoint.go:372 msg="writing series" size="603 kB"
level=debug ts=2026-05-09T15:41:53.670110376Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-09T15:41:55.89201094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:41:55.897631149Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T15:41:57.002407871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:41:57.002442885Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-09T15:41:58.591111283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:41:58.599738806Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:42:00.336490944Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:42:02.001944913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:03.669245504Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-09T15:42:07.001621185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:07.003141294Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:42:07.003156623Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=debug ts=2026-05-09T15:42:08.385972701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:42:08.441229234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:42:10.336092816Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-09T15:42:11.759092828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:42:11.764951079Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=info ts=2026-05-09T15:42:11.863304686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:42:11.866699272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:42:12.002208194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:13.669785179Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T15:42:13.958769581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:42:13.96596549Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:42:13.965983877Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:42:13.96600733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:42:13.972916471Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:42:17.002030427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:42:17.002048031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:17.002061443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:42:17.002069858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:42:17.003291119Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=info ts=2026-05-09T15:42:19.124863679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:42:19.133099639Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:42:20.336537536Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-09T15:42:22.002292117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:42:22.444156936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:42:22.448288958Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:42:23.108525329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:42:23.112271837Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-09T15:42:23.353032673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:42:23.669825256Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T15:42:27.002552048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:27.002599076Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-09T15:42:28.107368764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:42:28.112804166Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:42:28.262179505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:42:28.265576172Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:42:30.335879375Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T15:42:30.781322615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:42:30.790243614Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:42:31.472075581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:42:31.477757082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:42:32.002369667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:33.66930597Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=debug ts=2026-05-09T15:42:35.132006959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:42:37.002227021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:37.002830348Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=debug ts=2026-05-09T15:42:37.002848671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-09T15:42:37.520246506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:42:37.524377483Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:42:40.337908826Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:42:40.337956871Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T15:42:42.001692119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:43.66989041Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:42:46.437596321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:42:46.444255215Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:42:46.984791821Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:42:46.985236734Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:42:47.002586577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:42:47.002605724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:42:47.002615559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:42:47.002624395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:47.003016487Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=debug ts=2026-05-09T15:42:50.335834176Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-09T15:42:50.625320706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:42:50.626750135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T15:42:52.002476758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:42:53.354795532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T15:42:53.416729098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:42:53.423334984Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:42:53.669971442Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T15:42:57.002458934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:42:57.002476954Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=debug ts=2026-05-09T15:43:00.336654052Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T15:43:01.266425203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:43:01.275554899Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:43:01.589520558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:43:02.002505551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:43:02.970490773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33006"
level=debug ts=2026-05-09T15:43:03.66916827Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-09T15:43:06.362537059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:43:06.367324806Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:43:07.001831817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:07.004395072Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T15:43:07.004410092Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=8
level=info ts=2026-05-09T15:43:07.33892611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:43:07.342735274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:43:10.335904899Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-09T15:43:10.906620804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:43:10.910116591Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:43:11.089658159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:43:11.094937558Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:43:11.25332777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:43:11.257798407Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=warn ts=2026-05-09T15:43:12.002601429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:13.669363588Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-09T15:43:13.81043097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:43:13.815828215Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:43:17.002482984Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:43:17.002507963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:43:17.002516033Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:43:17.002508003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:17.002515814Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T15:43:18.235776311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:43:18.239580008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:43:20.336061445Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:43:22.002508682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:43:23.356428713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:43:23.669262757Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T15:43:27.00202339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:27.003207966Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-09T15:43:28.127247669Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:43:30.338261066Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-09T15:43:30.338283493Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=7
level=info ts=2026-05-09T15:43:31.56961415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:43:31.577306168Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:43:32.001995331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:33.669749159Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-09T15:43:36.555912052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:43:37.002209043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:37.002359971Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T15:43:38.822854413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:43:38.829394565Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:43:40.335647554Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=info ts=2026-05-09T15:43:40.869341844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:43:40.875856172Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:43:42.002406348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:43:43.515417848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:43:43.522307698Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:43:43.52232503Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:43:43.522346484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:43:43.52839292Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:43:43.669884243Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-09T15:43:46.984705038Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:43:46.984710988Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:43:47.002006025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:43:47.002021529Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:43:47.002028813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:43:47.002027036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:47.003281614Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-09T15:43:50.33611564Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:43:52.002377999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:43:53.358532645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:43:53.59962219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:43:53.669759837Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:43:57.002573629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:43:57.003087341Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:43:57.00310287Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-09T15:43:57.681473577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:43:57.687367881Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:43:59.163194758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:43:59.170220144Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:43:59.438111426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:43:59.443760055Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:44:00.336110044Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-09T15:44:00.881761596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:44:00.885877366Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:44:02.001914391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:44:02.97587721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47176"
level=debug ts=2026-05-09T15:44:03.669235276Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:44:04.500493905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:44:04.505892246Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:44:05.225868795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:44:05.231346951Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:44:07.001655441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:07.002781297Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:44:07.826082057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:44:07.830947167Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=debug ts=2026-05-09T15:44:10.336634404Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-09T15:44:12.001907373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:13.670495512Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-09T15:44:16.184357475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47466"
level=info ts=2026-05-09T15:44:17.001899806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:44:17.001927818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:44:17.001937766Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:44:17.001971334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:17.003351545Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:44:17.003363183Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T15:44:17.466325159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:44:17.470071679Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:44:20.336294201Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T15:44:22.00196918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:44:22.838863597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:44:22.842869085Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T15:44:23.360178217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:44:23.670125072Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:44:25.426027395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:44:25.432517973Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:44:27.002062988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:27.054939208Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T15:44:27.054963099Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=debug ts=2026-05-09T15:44:27.854910506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:44:30.335840388Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-09T15:44:32.002083592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:44:32.375861135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:44:32.383472869Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
ts=2026-05-09T15:44:32.977888581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34560"
level=debug ts=2026-05-09T15:44:33.669544679Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-09T15:44:35.879775017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:44:35.887174225Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:44:37.002688206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:37.003892975Z caller=checkpoint.go:372 msg="writing series" size="985 kB"
level=debug ts=2026-05-09T15:44:37.003920739Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-09T15:44:37.695660005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:44:39.43607527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:44:39.439726529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:44:40.338555703Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-05-09T15:44:42.002498086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:43.669624283Z caller=checkpoint.go:372 msg="writing series" size="436 kB"
level=debug ts=2026-05-09T15:44:43.66964942Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-09T15:44:43.784863583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:44:43.789984866Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T15:44:44.862876019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43506"
level=info ts=2026-05-09T15:44:46.984568269Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:44:46.985334988Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:44:47.001579169Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:44:47.00159886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:44:47.001609213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:44:47.001609284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:47.002790598Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T15:44:48.908984643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:44:48.913718915Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:44:50.339636906Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T15:44:50.339658725Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-05-09T15:44:52.002456087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:44:53.362012103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:44:53.670721168Z caller=checkpoint.go:372 msg="writing series" size="878 kB"
level=warn ts=2026-05-09T15:44:57.001744254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:44:57.002879073Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-09T15:44:58.030404126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:44:58.036260803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:44:58.036281956Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:44:58.036319376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:44:58.04117785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:44:58.213746561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:44:58.217270885Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:44:58.217286923Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:44:58.336499699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:44:58.340122892Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:45:00.336552117Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T15:45:02.002330147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:45:03.052366664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:45:03.056173353Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:45:03.517702729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:45:03.521381298Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:45:03.670133866Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:45:03.670152598Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-09T15:45:03.833510736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:45:04.739982029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:45:04.745087203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=warn ts=2026-05-09T15:45:07.001863603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:45:07.00303968Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-09T15:45:10.336140442Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-09T15:45:11.075462201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:45:11.079387767Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:45:11.439233542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:45:11.447655366Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:45:11.711312325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:45:11.71676772Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:45:12.002664658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:45:13.682715116Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-09T15:45:13.682748697Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=info ts=2026-05-09T15:45:14.460801156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:45:14.46942409Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:45:17.001765813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:45:17.001804916Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:45:17.001832703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:45:17.001844496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:45:17.017381756Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-09T15:45:17.018706686Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.016485871s
level=warn ts=2026-05-09T15:45:22.002180338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:45:23.36330339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T15:45:26.063034311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:45:26.070848387Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T15:45:27.002646979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:45:27.921534681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:45:27.92887969Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:45:32.001752896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:45:33.967304145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:45:33.971748565Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:45:33.971769502Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:45:33.971789493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:45:33.975314003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:45:36.492553067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:45:37.002545883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:45:37.573085876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:45:37.577828867Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:45:37.675375865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:45:39.709466665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:45:39.713386503Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:45:42.002376195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:45:44.869016498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38098"
level=info ts=2026-05-09T15:45:46.686744957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:45:46.69285849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:45:46.902639156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:45:46.907357769Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=info ts=2026-05-09T15:45:46.984570025Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:45:46.984831566Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-09T15:45:46.986068648Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T15:45:46.986118919Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975934 n=1
level=debug ts=2026-05-09T15:45:46.986162483Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975934 n=1
level=info ts=2026-05-09T15:45:47.002088085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-09T15:45:47.002087648Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T15:45:47.002106307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:45:47.002117656Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:45:47.002126474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:45:47.002210705Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-09T15:45:47.002219583Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:45:47.002237353Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.395µs success=true
level=debug ts=2026-05-09T15:45:47.003401735Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-09T15:45:47.003411979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
level=info ts=2026-05-09T15:45:47.068541718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:45:47.073745769Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:45:49.889799703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:45:49.893354247Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:45:50.33674529Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-09T15:45:52.002338202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:45:53.365122566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:45:53.669994123Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-09T15:45:55.388207968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:45:55.391947578Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:45:57.002405286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:45:57.002523891Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:45:57.263648897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:45:57.271230992Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:46:00.336344456Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:46:02.001877099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:46:02.984542319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52550"
level=debug ts=2026-05-09T15:46:03.626748564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:46:03.669971786Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:46:07.002012041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:07.003224299Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-05-09T15:46:07.987263452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:46:07.996700472Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:46:09.162440468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:46:09.168005525Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:46:10.336126728Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-09T15:46:12.001632483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:13.669364335Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T15:46:16.195315406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38724"
level=debug ts=2026-05-09T15:46:16.778560378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:46:17.0019897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:46:17.002016768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:46:17.002011688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:46:17.0020256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:46:17.017681025Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:46:17.01770056Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=info ts=2026-05-09T15:46:17.449478549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:46:17.453188581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:46:20.336733674Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-09T15:46:22.002529214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:46:22.891601126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:46:22.898770865Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T15:46:23.366438466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:46:23.670261102Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T15:46:27.002563759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:27.003036238Z caller=checkpoint.go:372 msg="writing series" size="520 kB"
level=debug ts=2026-05-09T15:46:30.347097887Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T15:46:30.34712048Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=4
level=info ts=2026-05-09T15:46:30.992860183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:46:30.996784116Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:46:31.077994071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:46:31.0813508Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:46:32.001910351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:46:32.98641338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48742"
level=debug ts=2026-05-09T15:46:33.669337802Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T15:46:37.002008893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:37.003117458Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T15:46:37.055395547Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:46:37.055372897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:46:37.058739932Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T15:46:37.356236925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="405 kB" avg_uncomp="405 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:46:37.359632016Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:46:37.432378065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:46:37.436787868Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:46:37.656099696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:46:37.659190177Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:46:37.955923433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:46:37.958962985Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:46:38.255921119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:46:38.259752384Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T15:46:38.270791584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:46:38.275436034Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=info ts=2026-05-09T15:46:38.556066442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:46:38.559383611Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:46:38.856356838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="132 kB" avg_uncomp="132 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:46:38.859542459Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:46:40.336856388Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-09T15:46:42.002423199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:43.669999833Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T15:46:44.396451077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:46:44.402111658Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T15:46:46.196802562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:49124"
level=info ts=2026-05-09T15:46:46.984535643Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:46:46.985476162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:46:47.00172847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:46:47.001747617Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:46:47.001757069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:46:47.001843573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:47.002915019Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-09T15:46:47.052302214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:46:47.060733205Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:46:49.164315255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:46:49.167633685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:46:49.326181152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:46:49.32937769Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:46:50.061491014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:46:50.067796695Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:46:50.336663373Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T15:46:52.002249466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:46:52.117676904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:46:52.120559562Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:46:53.286011803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:46:53.289576127Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T15:46:53.289595938Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:46:53.289617271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:46:53.292768098Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T15:46:53.368090181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:46:53.66998735Z caller=checkpoint.go:372 msg="writing series" size="648 kB"
level=debug ts=2026-05-09T15:46:53.670010226Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-05-09T15:46:53.754043252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:46:53.755764566Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:46:54.496207993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:46:54.499129578Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T15:46:55.21022468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:46:55.213265682Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:46:56.314109554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:46:56.317128573Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T15:46:57.002343876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:46:57.003555597Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-09T15:46:57.576609755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:46:57.579770913Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:47:00.336190814Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-09T15:47:01.166498862Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:47:01.166547817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:47:01.169649172Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:47:02.002448534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:47:02.982353528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T15:47:02.987888198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43896"
level=error ts=2026-05-09T15:47:02.98868414Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:47:03.670287408Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=warn ts=2026-05-09T15:47:07.002594576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:47:07.003353748Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:47:07.003369068Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T15:47:07.616415468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:47:07.620669171Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:47:08.343303247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:47:08.352857637Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:47:10.336189967Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-09T15:47:11.068404008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:47:11.072117209Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:47:12.002484069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:47:13.669851587Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T15:47:16.852869705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:47:16.856897111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:47:16.958245241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:47:16.961032558Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:47:17.002077174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:47:17.002096594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:47:17.002088103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:47:17.002104196Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:47:17.003604372Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=info ts=2026-05-09T15:47:18.374671177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:47:18.378360533Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T15:47:18.466625315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:47:18.470121275Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T15:47:20.336599638Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-09T15:47:22.00272093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:47:22.175061185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:47:22.178228666Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:47:22.99313876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:47:22.996152263Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T15:47:23.369054131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T15:47:23.482251001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:47:23.486522924Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-09T15:47:23.486542262Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:47:23.486564761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:47:23.490331303Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:47:23.669840818Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-09T15:47:25.182236238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5acf9614cea6348b immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:47:25.187196197Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5acf9614cea6348b/19e0d3a63dd%3A19e0d5b2372%3A9da64bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-caxu\"}"
level=error ts=2026-05-09T15:47:25.187214831Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:47:25.187234084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="853 B" avg_uncomp="853 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:47:25.189768956Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T15:47:25.464525704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:47:26.265723865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:47:26.269952678Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:47:26.269979051Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:47:26.270004638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:47:26.273348072Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:47:27.0025864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:47:27.002593916Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-09T15:47:27.31702485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:47:27.320081788Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:47:29.962069701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:47:30.336267574Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-09T15:47:32.001679048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:47:33.212854465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:47:33.216110351Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:47:33.441499344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:47:33.444378829Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T15:47:33.670003123Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-09T15:47:34.833392775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:47:34.837409856Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:47:35.223523959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:47:35.228503007Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:47:36.150924646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:47:36.157349391Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:47:37.001695341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:47:37.003056799Z caller=checkpoint.go:372 msg="writing series" size="594 kB"
level=debug ts=2026-05-09T15:47:37.003071869Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-05-09T15:47:37.871688487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:47:37.875186928Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:47:40.336876438Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:47:40.336898433Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T15:47:42.002266912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:47:42.98062279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:47:42.984452253Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:47:43.150866684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:47:43.154383271Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:47:43.669718606Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:47:46.984519837Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:47:46.984757327Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:47:47.001982049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:47:47.001990005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:47:47.002000691Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:47:47.002008553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:47:47.003402081Z caller=checkpoint.go:372 msg="writing series" size="538 kB"
level=info ts=2026-05-09T15:47:49.408767094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:47:49.416589923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:47:49.416616607Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:47:49.416642683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:47:49.422515985Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:47:49.85559308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:47:49.860454976Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:47:49.860476985Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:47:49.860500197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:47:49.864856277Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:47:50.336330874Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-09T15:47:52.002532023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:47:53.371120901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:47:53.67021081Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T15:47:56.66575227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:47:56.670351263Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:47:57.00241746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:47:57.002523543Z caller=checkpoint.go:372 msg="writing series" size="940 B"
level=info ts=2026-05-09T15:47:58.072510412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:47:58.075474704Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T15:47:58.36417143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:47:59.245752898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:47:59.252827767Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:47:59.598862916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:47:59.602786991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:47:59.74802116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:47:59.750783439Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:47:59.86216118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:47:59.865421018Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:48:00.336799357Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T15:48:02.002150663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:48:02.352678627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:48:02.359048229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:48:02.913577407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:48:02.918638319Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:48:03.212639055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:48:03.215322164Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:48:03.669898737Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-09T15:48:04.255195115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:48:04.258621892Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:48:07.001995003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:07.005410656Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T15:48:07.005426807Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=8
level=info ts=2026-05-09T15:48:08.112411621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:48:08.115262583Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:48:10.336869769Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:48:12.002054296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:13.670228427Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-09T15:48:14.323765574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:48:14.327080627Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T15:48:16.203257593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47332"
level=info ts=2026-05-09T15:48:17.002030886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:48:17.002059295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:48:17.002070364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:48:17.00207196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:17.003207939Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T15:48:18.282772993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:48:18.286249795Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:48:19.764924871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:48:19.768656448Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:48:20.336467792Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T15:48:22.001669144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:48:22.118002982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:48:22.120892562Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T15:48:23.373608432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:48:23.669857091Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T15:48:23.87445578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:48:23.877536152Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:48:25.184307742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:48:25.192109849Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:48:26.971248859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:48:26.974724229Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:48:27.001605977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:27.002682237Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-09T15:48:28.763734935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:48:29.863599444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:48:30.338992948Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-09T15:48:30.339017952Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=7
level=warn ts=2026-05-09T15:48:32.001689131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:48:32.588281369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:48:32.593195631Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:48:32.626250641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:48:32.630867382Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:48:33.6699725Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-09T15:48:37.001991663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:37.003218571Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T15:48:38.039403121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:48:38.045787683Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:48:40.335967616Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=info ts=2026-05-09T15:48:41.580989404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:48:41.584440484Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:48:42.002450375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:48:42.036588226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:48:42.039880782Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T15:48:43.669778186Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-09T15:48:44.747751441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:48:44.753415319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:48:45.715222531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:48:45.71890118Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:48:46.98454084Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:48:46.985336215Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:48:47.001572282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:48:47.001587948Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:48:47.00159834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:48:47.001630912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:47.002905082Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-05-09T15:48:49.962789092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:48:49.970904358Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:48:49.970926242Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:48:49.970954246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:48:49.977695027Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:48:50.33673732Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T15:48:52.002061475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:48:52.076828734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:48:52.08096875Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-09T15:48:53.376030791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:48:53.669576694Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-09T15:48:53.669598057Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T15:48:55.973788652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:48:55.977248237Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T15:48:57.001699257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:48:57.00307689Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:48:58.149558039Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:48:58.870534793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:48:58.873741891Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:48:59.189863849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:48:59.192736808Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:49:00.336754786Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-09T15:49:02.002453198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:49:02.773518227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:49:02.77761112Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:49:03.669390458Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:49:04.996986407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:49:05.000175863Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:49:05.393233982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:49:05.400073963Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:49:07.001995216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:07.003150384Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:49:08.184372842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:49:08.18722999Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T15:49:08.55437629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:49:08.561400652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:49:09.774028161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:49:10.336885802Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-09T15:49:11.376027921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:49:11.378873118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T15:49:12.00228154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:13.66982785Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
ts=2026-05-09T15:49:14.893754773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:46918"
level=info ts=2026-05-09T15:49:15.277709631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:49:15.28074417Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T15:49:17.001606468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:49:17.001619505Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:49:17.001642124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:49:17.001650535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:49:17.00296911Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:49:17.002979259Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=info ts=2026-05-09T15:49:19.408111207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:49:19.413230241Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:49:20.196530995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:49:20.200850303Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:49:20.336120003Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-09T15:49:22.002422722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:49:23.378241437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:49:23.669938199Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:49:24.513653265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:49:24.519355644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T15:49:27.002444314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:27.063746874Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-09T15:49:27.063772101Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=debug ts=2026-05-09T15:49:28.165549695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:49:30.336864957Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-09T15:49:30.625629128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:49:30.633042545Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:49:31.331093551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:49:31.337125866Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:49:32.001873563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:33.670267791Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-09T15:49:34.252690343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:49:34.259197174Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:49:35.470931536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:49:35.477472143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:49:36.143290589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:49:36.163307577Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:49:37.002210648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:37.005736689Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:49:37.005758507Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-09T15:49:39.813860924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:49:39.816977372Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T15:49:40.33770052Z caller=checkpoint.go:372 msg="writing series" size="919 kB"
level=info ts=2026-05-09T15:49:40.504382114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:49:40.511196941Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:49:42.002474623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:43.670631277Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=debug ts=2026-05-09T15:49:43.670654794Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
ts=2026-05-09T15:49:44.896574354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38496"
ts=2026-05-09T15:49:46.209555698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:46526"
level=info ts=2026-05-09T15:49:46.984531468Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:49:46.985123485Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:49:47.002492748Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:49:47.002508824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:49:47.002516107Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:49:47.002534172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:47.002552012Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T15:49:49.933175692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:49:49.936635441Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:49:50.339348999Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T15:49:50.339369398Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=debug ts=2026-05-09T15:49:50.794491052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:49:52.002302734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:49:52.046422847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:49:52.049572444Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-09T15:49:53.38009238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:49:53.670383658Z caller=checkpoint.go:372 msg="writing series" size="927 kB"
level=info ts=2026-05-09T15:49:54.178733758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:49:54.182513369Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:49:54.961354194Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:49:57.002005071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:49:57.003092855Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T15:49:57.91642636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:49:57.924353398Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:49:57.924372147Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:49:57.924397082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:49:57.931333078Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:49:58.452616229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:49:58.454449892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:49:58.45573605Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:49:58.455744567Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:49:59.115719923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:49:59.118684631Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:50:00.336182606Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T15:50:00.433683081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:50:00.436973246Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:50:00.549230215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:50:00.552234531Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T15:50:02.002414478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:50:03.669464687Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T15:50:03.669485074Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-09T15:50:07.001767282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:50:07.00290891Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-09T15:50:10.336556454Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T15:50:12.002105133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:12.729737468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:50:12.736119354Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:50:13.684792775Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:50:13.684853031Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=3
ts=2026-05-09T15:50:14.899699646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:47422"
level=info ts=2026-05-09T15:50:15.077940374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:50:15.083608826Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:50:17.002013628Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:50:17.002041972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:50:17.00205013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:50:17.002069605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:17.019857032Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-09T15:50:17.021332817Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.018918819s
level=info ts=2026-05-09T15:50:17.755522436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:50:17.763448547Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:50:18.785156915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:50:18.792776351Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:50:19.661105763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:50:19.664773134Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:50:22.002504799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:23.378495578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T15:50:23.381186672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=error ts=2026-05-09T15:50:23.382480883Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T15:50:23.849692075Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:50:24.47637172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:50:24.483058192Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:50:24.730677991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:50:24.734514633Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:50:25.090634641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:50:25.093446012Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:50:27.002432847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:28.336776461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:50:28.345045616Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:50:32.002310171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:32.93958239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:50:32.943602819Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:50:37.002498224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:50:42.001906866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:50:44.902494777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:32934"
level=info ts=2026-05-09T15:50:45.105520766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:50:45.109426867Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:50:45.925256051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:50:45.928561285Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:50:46.098228676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:50:46.100773623Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:50:46.682003186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:50:46.685408086Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:50:46.984554666Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:50:46.984794723Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:50:47.002070518Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T15:50:47.002069639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:47.002099132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:50:47.002125043Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:50:47.002134547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:50:47.002210666Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-09T15:50:47.002219175Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:50:47.002235762Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.301µs success=true
level=debug ts=2026-05-09T15:50:47.003593804Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-09T15:50:47.003603111Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
level=debug ts=2026-05-09T15:50:47.095237142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:50:47.096714465Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T15:50:47.279297818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:50:47.285770691Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:50:49.480163633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:50:49.483961573Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:50:50.33627922Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T15:50:50.658194839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:50:50.661227785Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T15:50:50.919066444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:50:52.00162459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:50:52.584314509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:50:52.58718808Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T15:50:53.382495158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:50:53.670256905Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-09T15:50:57.002399989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:50:57.002495348Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:50:57.627379724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:50:57.630584384Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:51:00.33678406Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T15:51:00.433125092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:51:00.438964613Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T15:51:02.002162432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:51:03.066521454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:51:03.071089278Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:51:03.669591273Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:51:05.492618283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:51:05.499133027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:51:05.93125199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:51:05.937711475Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:51:06.897710696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:51:06.900768725Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T15:51:07.002004196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:51:07.003400437Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=debug ts=2026-05-09T15:51:10.336011032Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:51:11.367877575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:51:11.371678957Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:51:11.371702288Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:51:11.371726085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:51:11.375238783Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:51:12.002584444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:51:12.582142518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:51:12.588968665Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:51:13.669222669Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T15:51:14.906218171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53324"
ts=2026-05-09T15:51:16.216344654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48466"
level=warn ts=2026-05-09T15:51:17.002441407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:51:17.002457313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:51:17.002490602Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:51:17.002499857Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:51:17.012802402Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:51:17.012836553Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=debug ts=2026-05-09T15:51:20.336598796Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-09T15:51:21.859589585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:51:21.864452134Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:51:22.002631381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:51:23.384199506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T15:51:23.430178618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:51:23.434320568Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:51:23.670108258Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:51:25.45736343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:51:25.461315209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:51:26.806278468Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:51:27.001651411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:51:27.002997159Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=debug ts=2026-05-09T15:51:30.351062024Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T15:51:30.351084738Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=4
level=info ts=2026-05-09T15:51:30.432286254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:51:30.435861948Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:51:32.002625449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:51:32.135618344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:51:32.139337157Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:51:33.314567568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:51:33.317830079Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T15:51:33.669691602Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-09T15:51:35.907788527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:51:35.910904939Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T15:51:37.001723442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:51:37.002835248Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-09T15:51:37.631448129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:51:37.63890347Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:51:40.336226637Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-09T15:51:41.638756516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:51:41.641829285Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T15:51:42.002547723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:51:42.675729833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:51:42.679183462Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:51:43.670401104Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T15:51:45.125336878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:51:45.128773314Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:51:46.984657042Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:51:46.985098363Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:51:47.002457707Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:51:47.002475353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:51:47.002483096Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:51:47.002497021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:51:47.00254438Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-09T15:51:47.026919547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:51:47.033204917Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:51:47.256324686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:51:47.259062356Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:51:48.742174357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:51:49.036257517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:51:49.039077371Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:51:50.336832576Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-05-09T15:51:52.001654585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:51:52.748686432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:51:52.755789777Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:51:52.862162458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:51:52.865465431Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T15:51:53.379591798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T15:51:53.385782616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:51:53.670505607Z caller=checkpoint.go:372 msg="writing series" size="656 kB"
level=debug ts=2026-05-09T15:51:53.670527444Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=warn ts=2026-05-09T15:51:57.001672316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:51:57.002939876Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=info ts=2026-05-09T15:51:57.429669922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:51:57.438163439Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:51:57.43818704Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:51:57.438219893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:51:57.442949919Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:51:57.472735166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:51:57.475702244Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:51:59.157214022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:51:59.16012471Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:52:00.326584326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:52:00.333842877Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:52:00.335891121Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T15:52:02.002635253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:52:03.018451134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53638"
level=debug ts=2026-05-09T15:52:03.669548884Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=info ts=2026-05-09T15:52:05.504735022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:52:05.508340166Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:52:07.002313178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:07.003958298Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:52:07.003971911Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T15:52:09.483858889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:52:09.487663721Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:52:09.48768482Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:52:09.487706553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:52:09.490617635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:52:10.336463656Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=warn ts=2026-05-09T15:52:12.00264542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:13.66941916Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T15:52:14.375583372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:52:14.381054276Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T15:52:14.912982549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45296"
level=info ts=2026-05-09T15:52:15.196754393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:52:15.199887114Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:52:15.715101488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:52:15.719174785Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:52:17.001695741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:52:17.00172262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:52:17.001731592Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:52:17.00176804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:17.003097182Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=info ts=2026-05-09T15:52:20.05398283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:52:20.058228122Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:52:20.066355277Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:52:20.336169237Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-09T15:52:22.002312308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:52:22.207226526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:52:22.214650871Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:52:22.833755028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:52:22.839028284Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T15:52:23.387846989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:52:23.669422845Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T15:52:27.002375213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:27.002438392Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-09T15:52:28.804240479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:52:28.807538498Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T15:52:30.33663215Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-09T15:52:32.002332349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:52:32.519620893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:52:32.523290046Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:52:33.04732095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:52:33.050590993Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T15:52:33.670005842Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=info ts=2026-05-09T15:52:34.014071454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:52:34.017604598Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T15:52:37.002514881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:37.002849317Z caller=checkpoint.go:372 msg="writing series" size="634 kB"
level=debug ts=2026-05-09T15:52:37.002865864Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=debug ts=2026-05-09T15:52:40.336854017Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:52:40.336877665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T15:52:40.698504769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:52:40.701532861Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:52:41.903129824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T15:52:42.002053917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:43.669338334Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:52:44.84086171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:52:44.846173682Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:52:44.847156221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:52:44.850365155Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T15:52:44.916212324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:32958"
level=info ts=2026-05-09T15:52:45.899509512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:52:45.906917011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:52:46.143529078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:52:46.146548444Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:52:46.249841145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:52:46.25257499Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:52:46.252589466Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:52:46.252608788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:52:46.254861329Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:52:46.755177226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:52:46.757939424Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T15:52:46.984691217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:52:46.984750054Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:52:47.001978168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:52:47.00199639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:52:47.002004782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:52:47.002059327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:47.003536175Z caller=checkpoint.go:372 msg="writing series" size="641 kB"
level=info ts=2026-05-09T15:52:48.944383613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:52:48.948234193Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:52:50.194370353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:52:50.202107452Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:52:50.336416152Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-09T15:52:50.740572582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:52:50.740632127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:52:50.743550232Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=warn ts=2026-05-09T15:52:52.002570761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:52:53.389261653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:52:53.669632537Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T15:52:55.805128998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:52:55.811936179Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:52:56.385605554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:52:56.389151636Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:52:57.002118415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:52:57.003135799Z caller=checkpoint.go:372 msg="writing series" size="907 B"
level=info ts=2026-05-09T15:52:58.116937449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:52:58.120917592Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T15:53:00.336009169Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T15:53:01.779228961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:53:01.782635567Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T15:53:02.002332407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:53:02.558601126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:53:02.561560651Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:53:02.561582002Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:02.561603177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:53:02.564257174Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T15:53:03.669973633Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-09T15:53:06.326630778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="634 B" avg_comp="634 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:53:06.330853287Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=warn ts=2026-05-09T15:53:07.00175233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:53:07.004330768Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T15:53:07.004348886Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=8
level=info ts=2026-05-09T15:53:09.628176357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:53:09.631514004Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:53:09.631541964Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:09.631565723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:53:09.634087731Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:53:10.336036292Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:53:12.002635031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:53:13.669598538Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=info ts=2026-05-09T15:53:14.665185135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:53:14.674099265Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:53:16.881674844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:53:16.884800034Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T15:53:16.884842081Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:16.884864694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:53:16.88718771Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T15:53:17.002445254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:53:17.002466911Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:53:17.002478877Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-09T15:53:17.002487034Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:53:17.002494657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:53:18.220512393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:53:19.680403184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:53:19.683594494Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:53:20.336370935Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T15:53:21.979588582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:53:21.983269051Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T15:53:22.002454414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:53:23.202436229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:53:23.205626644Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T15:53:23.390600883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:53:23.669915815Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T15:53:25.36806113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:53:25.373359362Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T15:53:25.373379244Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:25.37340129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:53:25.376616039Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:53:26.984152369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:53:26.987018227Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:53:26.987036533Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:26.987057419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:53:26.989642463Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T15:53:27.001731333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:53:27.003024239Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-09T15:53:28.010942705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:53:28.014894096Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:53:30.339733389Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-09T15:53:30.339756839Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=7
level=warn ts=2026-05-09T15:53:32.002661783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:53:32.220393117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:53:32.22357554Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T15:53:32.858848933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:53:32.864908572Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:53:32.953083578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:53:32.95588841Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T15:53:33.669652351Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T15:53:35.257026586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:53:35.259698275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:53:35.259717253Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:35.259737545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:53:35.261909004Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:53:35.838366485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T15:53:35.960593775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:53:35.964081488Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:53:36.290315312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:53:36.293108128Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T15:53:37.001961817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:53:37.003169753Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T15:53:38.51720033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:53:38.523941877Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:53:40.075397534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:53:40.078456069Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:53:40.078476226Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:40.078498165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:53:40.080868864Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T15:53:40.33651109Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:53:42.002521925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:53:42.189103005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:53:42.192113077Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:53:43.187751441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:53:43.194190471Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:53:43.376552366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:53:43.379545441Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:53:43.379564482Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:43.37958843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:53:43.382007451Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:53:43.669857232Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-09T15:53:43.983733766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:53:43.989165565Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:53:44.980515998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:53:44.984226014Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:53:46.160380017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:53:46.164400941Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:53:46.984545669Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:53:46.984560673Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:53:47.001843394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:53:47.001867749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:53:47.001878207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:53:47.001887147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:53:47.003208266Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=info ts=2026-05-09T15:53:47.284473628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:53:47.286908995Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T15:53:47.289907392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:53:47.292555705Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:53:50.086860975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:53:50.089650325Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T15:53:50.336307849Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T15:53:50.903924462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:53:50.906697326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:53:51.72584189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:53:52.002291063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:53:53.392513146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:53:53.670209803Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=debug ts=2026-05-09T15:53:53.670234635Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-09T15:53:54.461934063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:53:54.465874155Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-09T15:53:54.46589237Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:53:54.465912568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:53:54.469079463Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:53:57.002584223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:53:57.002899647Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-09T15:53:59.211698299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:53:59.216319018Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:54:00.336245981Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-09T15:54:02.002577133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:54:02.638721629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:54:02.642056361Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:54:03.669566656Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-09T15:54:06.23399715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:54:06.237946837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:54:06.58299719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:54:06.586753922Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:54:07.001648184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:54:07.002707741Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:54:07.757111352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:54:07.759970532Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:54:08.269923917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:54:08.272956469Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:54:08.272978566Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:54:08.27300104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:54:08.275353121Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T15:54:10.336175829Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-09T15:54:11.417428874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:54:11.420955657Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:54:11.420975532Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:54:11.420999079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:54:11.423860042Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:54:12.001857834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:54:12.085175713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:54:12.091775374Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T15:54:13.669331685Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-09T15:54:17.001697712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:54:17.001731263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:54:17.001741446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:54:17.003883291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:54:17.004243672Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:54:17.004261749Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=debug ts=2026-05-09T15:54:17.365999671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:54:17.679333113Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:54:18.195769997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:54:18.199291646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:54:20.336339715Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-09T15:54:20.498931956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:54:20.502138984Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:54:20.936276729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:54:20.939160007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:54:21.677249051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:54:21.68068899Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:54:21.791057401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:54:21.793919336Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T15:54:22.00231409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:54:23.394380174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:54:23.670028764Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:54:26.353546016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:54:26.354452807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:54:26.357983532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:54:26.358021517Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:54:27.001819941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:54:27.085897147Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-09T15:54:27.085931402Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-09T15:54:27.170257606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:54:27.175749402Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:54:29.633640824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:54:29.642027108Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T15:54:30.336553889Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:54:32.001994895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:54:33.031611353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54850"
level=debug ts=2026-05-09T15:54:33.669740242Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:54:34.923864998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:54:34.94076113Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:54:34.940800518Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:54:34.940866427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:54:34.954178229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T15:54:37.001870667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:54:37.005505077Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:54:37.005532143Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-09T15:54:37.160559725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:54:37.164860227Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:54:38.294004286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="74 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:54:38.302219403Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:54:39.592530996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:54:39.598313055Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:54:39.598340759Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:54:39.598365905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:54:39.603343245Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T15:54:40.337523789Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:54:40.337546828Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T15:54:40.571927871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:54:40.575174266Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T15:54:42.00201553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:54:42.365063611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:54:42.369148058Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:54:43.130047429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:54:43.133668249Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:54:43.669767668Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=info ts=2026-05-09T15:54:43.937209023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:54:43.940221657Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T15:54:44.929051542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53880"
level=info ts=2026-05-09T15:54:46.925132305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:54:46.932372008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:54:46.984513227Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:54:46.984847818Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=warn ts=2026-05-09T15:54:47.00216081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:54:47.002179649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:54:47.002196384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:54:47.002204737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:54:47.003302704Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-09T15:54:50.339481011Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T15:54:50.339503539Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=info ts=2026-05-09T15:54:50.827358588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:54:50.831911014Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T15:54:52.002040893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:54:52.361601928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T15:54:52.361656406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-09T15:54:52.364996076Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T15:54:52.41252945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:54:52.417772989Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T15:54:53.396011769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:54:53.669627391Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=info ts=2026-05-09T15:54:55.651131861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:54:55.654495247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T15:54:55.848075367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:54:55.851928908Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T15:54:57.002348511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:54:57.003435617Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-09T15:55:00.33618826Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-09T15:55:00.336212553Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-09T15:55:00.571212954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:55:00.572758787Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T15:55:02.001678589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:55:03.545925023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:55:03.54884377Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:55:03.669374924Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=info ts=2026-05-09T15:55:03.840761521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:55:03.843734162Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:55:03.888917251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:55:03.892219868Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:55:06.511934284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:55:06.51502195Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T15:55:07.001839929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:55:07.002939176Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-09T15:55:07.028009674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:55:07.030877449Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T15:55:07.673090602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:55:07.679673119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:55:07.916298373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:55:07.919041626Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:55:10.065753771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:55:10.068740701Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T15:55:10.336530684Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T15:55:12.002220594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:55:13.68389979Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T15:55:13.683923546Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
ts=2026-05-09T15:55:16.239381119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:59620"
level=info ts=2026-05-09T15:55:17.002010549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:55:17.002050599Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:55:17.002061082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:55:17.002059006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:55:17.020567521Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-09T15:55:17.022127804Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.019743021s
level=debug ts=2026-05-09T15:55:17.518044168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:55:18.997833048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:55:19.00208111Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:55:19.294936204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:55:19.300917859Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:55:19.914477533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:55:19.921549075Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:55:21.3036316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:55:21.308872264Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T15:55:22.001936955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:55:23.397929287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T15:55:23.484188283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:55:23.487648607Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:55:25.669705402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:55:25.673270831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:55:27.002445826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:55:29.287512503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:55:29.290523214Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T15:55:30.73622791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:55:30.740978481Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T15:55:31.440214597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:55:31.443190898Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T15:55:32.002273679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:55:33.034990608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55920"
level=info ts=2026-05-09T15:55:34.785866463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:55:34.792884248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:55:34.792904799Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:55:34.792931158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:55:34.799749822Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:55:34.83482485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:55:34.837341625Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:55:35.174254653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:55:35.178201162Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:55:37.001650425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T15:55:42.002052215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:55:43.774996871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:55:43.779056918Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T15:55:44.936406112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35888"
level=info ts=2026-05-09T15:55:46.984516133Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:55:46.98577435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:55:47.002008881Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T15:55:47.002009306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:55:47.002017894Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:55:47.002038747Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:55:47.002049992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T15:55:47.002138137Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-09T15:55:47.00214657Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T15:55:47.002162568Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.383µs success=true
level=debug ts=2026-05-09T15:55:47.003904074Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-09T15:55:47.003914257Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
level=info ts=2026-05-09T15:55:47.453049005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:55:47.457228118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:55:48.170484724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:55:48.173679443Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:55:49.362092182Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:55:50.336036881Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T15:55:50.813467068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:55:50.825215933Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:55:52.001657387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:55:52.055860893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:55:52.059953176Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:55:53.047408525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:55:53.050224143Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-09T15:55:53.399991116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:55:53.669866961Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=warn ts=2026-05-09T15:55:57.001855857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:55:57.002917645Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-09T15:55:58.406457676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:55:58.409585551Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T15:56:00.336292286Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T15:56:02.002677846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:56:02.193062487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:56:02.195654611Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:56:02.726300733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:56:02.728982452Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:56:02.799867948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:56:03.670050532Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:56:03.916688925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:56:03.921495153Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:56:05.455191685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:56:05.458470148Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:56:06.555527345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:56:06.562303466Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:56:07.002131833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:56:07.00329178Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-09T15:56:09.671046031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:56:09.677158329Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:56:10.336134968Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-09T15:56:12.002433239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:56:12.156601015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:56:12.159884681Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T15:56:13.669374947Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-09T15:56:13.899806326Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T15:56:14.940100885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:34016"
ts=2026-05-09T15:56:16.242300744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34394"
level=info ts=2026-05-09T15:56:17.002439465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:56:17.002466428Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T15:56:17.002459291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:56:17.002475193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:56:17.016849164Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T15:56:17.01686779Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=info ts=2026-05-09T15:56:17.219828364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:56:17.224354428Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:56:17.264400103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:56:17.268754895Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:56:18.066503281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:56:18.069475296Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T15:56:18.27822415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:56:18.285553857Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:56:20.335953876Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-09T15:56:22.001686312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:56:22.345341383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:56:22.349290409Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:56:23.336712818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:56:23.339702963Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T15:56:23.401108553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:56:23.669474079Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T15:56:26.811300085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:56:26.81900891Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:56:27.002550934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:56:27.003095086Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=info ts=2026-05-09T15:56:27.433342737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:56:27.4371778Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:56:28.916542296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:56:28.919405132Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T15:56:30.344751811Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-09T15:56:30.344775862Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=4
level=warn ts=2026-05-09T15:56:32.002625164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:56:33.039178745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38590"
level=debug ts=2026-05-09T15:56:33.669275763Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-09T15:56:35.189922916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:56:35.194174775Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:56:37.001925612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:56:37.002976123Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-09T15:56:38.756530724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:56:38.759714376Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:56:40.3362488Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-09T15:56:42.00271957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:56:43.670057559Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T15:56:43.863246886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:56:43.866239455Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T15:56:44.878082611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:56:44.883084031Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T15:56:44.943051379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35664"
level=debug ts=2026-05-09T15:56:46.117196247Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:56:46.984592514Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:56:46.985300217Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:56:47.001602484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:56:47.001618351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:56:47.001626133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:56:47.00163437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:56:47.002715611Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=info ts=2026-05-09T15:56:47.05807663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:56:47.061215379Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T15:56:47.273162638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:56:47.276147723Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:56:48.680971737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:56:48.68409276Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:56:50.336422764Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-09T15:56:51.603925723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:56:51.611223613Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T15:56:51.611243056Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:56:51.611268056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:56:51.618343019Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=warn ts=2026-05-09T15:56:52.002058382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:56:52.650035736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:56:52.654452014Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-09T15:56:53.401923413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:56:53.669662301Z caller=checkpoint.go:372 msg="writing series" size="709 kB"
level=debug ts=2026-05-09T15:56:53.669687656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=warn ts=2026-05-09T15:56:57.002544258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:56:57.002641869Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=info ts=2026-05-09T15:56:58.789670754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:56:58.792905899Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T15:56:59.625940356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T15:57:00.335884129Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T15:57:01.056562413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:57:01.060155081Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:57:01.870332401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:57:01.873387339Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T15:57:02.002472604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:03.669926096Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=info ts=2026-05-09T15:57:04.093724519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:57:04.09818282Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:57:07.002198411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:07.004046467Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T15:57:07.004062504Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T15:57:07.158624099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:57:07.166024198Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:57:08.226522438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:57:08.232350989Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:57:08.347531612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:57:08.355960674Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:57:09.905588002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:57:09.908687972Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:57:10.336208504Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-09T15:57:12.001976496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:57:12.386692857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:57:12.389986768Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T15:57:13.669835079Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T15:57:15.441336755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:57:15.448882935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:57:15.933016547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:57:15.93619575Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T15:57:16.357370132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:57:16.360197542Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:57:17.001925304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:57:17.001955383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:57:17.001966563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:57:17.00195924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:17.003361333Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=info ts=2026-05-09T15:57:18.560299762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:57:18.563962942Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:57:18.832860753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:57:18.835943271Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:57:19.125898414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:57:20.336320144Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-09T15:57:22.002475006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:57:23.403398596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:57:23.669869631Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T15:57:27.001625387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:27.002744291Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-09T15:57:28.904579905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:57:28.909160039Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T15:57:30.336223636Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T15:57:31.52464729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:57:31.528197118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:57:32.002478163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:57:33.044028383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33182"
level=debug ts=2026-05-09T15:57:33.669786142Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=info ts=2026-05-09T15:57:33.812326186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:57:33.815551145Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T15:57:34.260778075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:57:34.268500862Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:57:35.396581233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:57:35.402275252Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T15:57:36.317716808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:57:36.320702072Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T15:57:37.002435591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:37.00257591Z caller=checkpoint.go:372 msg="writing series" size="445 kB"
level=debug ts=2026-05-09T15:57:37.002603257Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-05-09T15:57:40.337195711Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:57:40.337218833Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T15:57:40.352069898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:57:40.355906677Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:57:40.412311763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:57:40.415122061Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:57:42.002008509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:43.669870704Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T15:57:43.729012541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:57:43.732718757Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:57:43.732737383Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:57:43.73275835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:57:43.735748976Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T15:57:46.984542984Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:57:46.985090261Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:57:47.0024578Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:57:47.002474881Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:57:47.002485038Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:57:47.002494927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:47.002725051Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=info ts=2026-05-09T15:57:47.417662481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:57:47.420645765Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:57:49.283962176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:57:49.292648829Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T15:57:50.336206443Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-09T15:57:52.001876811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:52.750392146Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T15:57:53.356622563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:57:53.405109007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:57:53.669441841Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T15:57:54.962820716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:57:54.965762311Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T15:57:55.495418538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:57:55.501638763Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:57:55.506690279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:57:55.509042157Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T15:57:56.942006997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:57:56.945970396Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T15:57:57.00173957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:57:57.002838752Z caller=checkpoint.go:372 msg="writing series" size="907 B"
level=info ts=2026-05-09T15:58:00.190853301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:58:00.195599379Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:58:00.335880884Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T15:58:01.090197227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:58:01.093166631Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T15:58:02.002115071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:58:03.046150874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50532"
level=debug ts=2026-05-09T15:58:03.669298389Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-05-09T15:58:04.769381789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:58:04.774748887Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T15:58:06.348059631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:58:06.35201929Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T15:58:07.001861095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:07.00553273Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T15:58:07.005548399Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=8
level=info ts=2026-05-09T15:58:09.752926245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:58:09.756330715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T15:58:10.336212133Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T15:58:12.002602289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:13.669317058Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-09T15:58:14.732366591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:58:14.736630939Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T15:58:14.952864004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:43034"
level=info ts=2026-05-09T15:58:15.947422382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:58:15.950781083Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:58:17.001596784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:58:17.001623195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:58:17.001631476Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:58:17.001629577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:17.002760019Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T15:58:17.99567992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:58:17.998878292Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:58:19.87038458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:58:19.874371336Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:58:19.874387956Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:58:19.874409157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:58:19.877500125Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:58:20.335722668Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T15:58:20.862266447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:58:20.865925122Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T15:58:22.002425374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:22.293962026Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T15:58:23.407085483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T15:58:23.669230123Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T15:58:26.836847901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:58:26.842390542Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T15:58:27.001820943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:27.003000764Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-09T15:58:27.391696021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:58:27.394904753Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:58:29.010925977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="634 B" avg_comp="634 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:58:29.013722878Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T15:58:30.339212295Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-09T15:58:30.339235751Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=7
level=warn ts=2026-05-09T15:58:32.00256027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:33.669512979Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T15:58:33.690452986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:58:33.698517944Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:58:34.474512973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:58:34.477608548Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T15:58:34.97993792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:58:34.982901749Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T15:58:37.002610674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:37.002661239Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T15:58:37.567921199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:58:37.57076019Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T15:58:38.748105104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:58:38.751830911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T15:58:40.336146192Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T15:58:42.001661043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:58:43.352081886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:58:43.355097493Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T15:58:43.67015764Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-09T15:58:43.727260725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T15:58:44.956271661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33754"
level=info ts=2026-05-09T15:58:45.038079924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:58:45.045160372Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T15:58:45.672743223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:58:45.680219794Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T15:58:46.984662634Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T15:58:46.985078604Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:58:47.002452256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T15:58:47.002699912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:47.002955123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:58:47.002972619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T15:58:47.003337307Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
level=info ts=2026-05-09T15:58:47.42156568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:58:47.425457318Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:58:49.590009577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:58:49.593105693Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T15:58:50.336025108Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T15:58:51.364923916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:58:51.367939198Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T15:58:52.002673867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:58:53.256096697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:58:53.25902119Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T15:58:53.40980338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:58:53.670050819Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-09T15:58:53.670073472Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-09T15:58:54.348981099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:58:54.352501357Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T15:58:54.861932637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:58:54.867071993Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:58:54.867091773Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:58:54.867131492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:58:54.871594236Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:58:56.748931436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T15:58:57.00163939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:58:57.003054116Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T15:59:00.33646864Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T15:59:00.739577722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:59:00.743256135Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:59:00.996195394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:59:01.00028974Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T15:59:02.001659106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:59:02.041861849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:59:02.045205604Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:59:03.669886784Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-09T15:59:07.002505288Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-09T15:59:07.002509107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:59:08.532311917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:59:08.539355852Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:59:10.33684985Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-05-09T15:59:10.620788618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T15:59:10.624565374Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T15:59:10.713925859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:59:10.717028306Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T15:59:12.002330556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:13.669491144Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T15:59:14.69099598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T15:59:14.694265534Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T15:59:17.002451795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:59:17.002479357Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:59:17.002486959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:59:17.002507264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:17.00309917Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T15:59:17.003111346Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=debug ts=2026-05-09T15:59:20.336474845Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-09T15:59:20.580155113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T15:59:20.588121918Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T15:59:21.251868225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T15:59:21.257529397Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T15:59:22.002140168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:59:23.411058208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T15:59:23.669692064Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T15:59:26.334230601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T15:59:26.337305948Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T15:59:27.002228415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:27.074881904Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-09T15:59:27.074910587Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-09T15:59:27.141992102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:59:27.149708355Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:59:27.149740537Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:27.149777814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:59:27.157896179Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T15:59:29.220116694Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T15:59:29.47278247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T15:59:29.483331424Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T15:59:30.337016735Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T15:59:30.982890868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T15:59:30.997067283Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T15:59:32.00218471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:33.669688759Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-09T15:59:34.691733737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:59:34.700389436Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:59:34.700413965Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:34.700446631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:59:34.714773926Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:59:34.950032986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T15:59:34.959835556Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T15:59:36.261715441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:59:36.268950027Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:59:36.268984213Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:36.269021746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:59:36.275230197Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T15:59:37.001662149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:37.005633615Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T15:59:37.005660775Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-09T15:59:37.513407739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T15:59:37.517687346Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T15:59:38.253036183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T15:59:38.2585246Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T15:59:40.337771538Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=info ts=2026-05-09T15:59:40.458025192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T15:59:40.46615469Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T15:59:40.58654825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T15:59:40.588282138Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T15:59:42.002247905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T15:59:42.440604038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T15:59:42.444619783Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T15:59:42.725280614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T15:59:42.728787396Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T15:59:43.669643231Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-09T15:59:43.669666028Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-09T15:59:43.947040194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:59:43.95008017Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T15:59:43.950100521Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:43.950132049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:59:43.952436654Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T15:59:44.633377977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T15:59:44.636550585Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T15:59:46.184765129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T15:59:46.18868103Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T15:59:46.259483905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54298"
level=info ts=2026-05-09T15:59:46.599766151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T15:59:46.603410024Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T15:59:46.984446157Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T15:59:46.985512297Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T15:59:47.001777867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T15:59:47.00179704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T15:59:47.001805404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T15:59:47.001837827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:47.002935196Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T15:59:47.05118442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:59:47.054195582Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:59:47.054218453Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:47.054243516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T15:59:47.056772115Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T15:59:50.338638799Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T15:59:50.338662561Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-05-09T15:59:52.002517219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T15:59:53.413017341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T15:59:53.670052427Z caller=checkpoint.go:372 msg="writing series" size="860 kB"
level=info ts=2026-05-09T15:59:53.933226165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:59:53.936484801Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:59:53.936503385Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:53.936524065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T15:59:53.938978017Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T15:59:54.181990866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:59:54.18560434Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T15:59:54.185621611Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T15:59:54.236798235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T15:59:54.240086606Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T15:59:54.360463404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T15:59:54.363493162Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T15:59:56.46950393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T15:59:56.472443402Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T15:59:57.002593286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T15:59:57.002625933Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-09T16:00:00.336535847Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T16:00:02.001946726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:02.616530123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:00:02.619703271Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T16:00:03.056531262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41618"
level=debug ts=2026-05-09T16:00:03.104497039Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:00:03.669949444Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T16:00:03.669972349Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-09T16:00:06.354447078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:00:06.357958025Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:00:06.357976609Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:00:06.358003118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:00:06.360478841Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T16:00:07.002132856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:00:07.003226937Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-09T16:00:07.450922979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:00:07.453867655Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:00:07.61208707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T16:00:07.619939694Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T16:00:08.571497082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:00:08.576401573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:00:09.222261751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:00:09.225306975Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:00:09.225327696Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:00:09.225347261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:00:09.228363449Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:00:09.671478717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:00:09.674537192Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:00:10.336644003Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T16:00:12.002357095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:12.525321621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:00:12.529021414Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T16:00:13.683121721Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T16:00:13.683148968Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-09T16:00:15.980758952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:00:15.986772794Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:00:17.002392148Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:00:17.00241715Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:00:17.002425185Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:00:17.002418006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:17.018970837Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-09T16:00:17.020526751Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.018233002s
level=info ts=2026-05-09T16:00:17.453518634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:00:17.456330022Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:00:19.936515176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:00:19.939859719Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T16:00:21.576923868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:00:21.58040582Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T16:00:22.00187714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:00:23.413965498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:00:23.561860644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:00:23.569086726Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:00:24.844010996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:00:24.846918173Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:00:24.846939486Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:00:24.84696233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:00:24.849267512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:00:25.694544264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:00:25.697440723Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:00:26.125689342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:00:26.131140217Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:00:27.002404371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:27.398170269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:00:27.400953492Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:00:28.060418732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:00:28.063419298Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:00:32.0026867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:00:32.332246127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:00:33.353797819Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:00:33.50328418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:00:33.506907896Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:00:35.235006458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:00:35.237960577Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:00:35.327163617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:00:35.334072965Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:00:35.961538381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:00:35.965895758Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-09T16:00:35.965915262Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:00:35.965938966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:00:35.969605672Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T16:00:36.232567937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:00:36.235466587Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T16:00:37.00175976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:39.70325988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:00:39.707217014Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:00:40.342936064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:00:40.3468341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T16:00:42.002059605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:00:44.968709505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:54860"
level=info ts=2026-05-09T16:00:45.341209297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:00:45.345951604Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:00:45.345970732Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:00:45.345992322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:00:45.350639389Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:00:46.984787792Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:00:46.985244272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-09T16:00:46.986479498Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T16:00:46.986523522Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975935 n=1
level=debug ts=2026-05-09T16:00:46.986564055Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975935 n=1
level=info ts=2026-05-09T16:00:47.002565244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:00:47.002594217Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:00:47.00260208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T16:00:47.002610029Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T16:00:47.002644553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:47.0027042Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-09T16:00:47.00271103Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T16:00:47.002729205Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.449µs success=true
level=debug ts=2026-05-09T16:00:47.004307703Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-09T16:00:47.004317617Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=info ts=2026-05-09T16:00:47.649512574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:00:47.654194443Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T16:00:50.336546707Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T16:00:50.608622762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:00:50.613154478Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T16:00:52.002325968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:00:52.543869163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:00:52.547328346Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:00:52.547351628Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:00:52.547378336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:00:52.549978813Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T16:00:53.415018585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:00:53.547931105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:00:53.551476289Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T16:00:53.670087589Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T16:00:56.272415122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:00:56.275370952Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T16:00:57.001975505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:00:57.00311041Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-09T16:00:58.270724711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:00:58.273528577Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:00:59.755580257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T16:00:59.762469676Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=info ts=2026-05-09T16:00:59.953010562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:00:59.955844076Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T16:01:00.337029627Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-09T16:01:01.5035927Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T16:01:02.001670975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:01:03.061853029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54522"
level=info ts=2026-05-09T16:01:03.592869038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:01:03.599065444Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:01:03.599084751Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:01:03.599107793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:01:03.603654444Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:01:03.669954624Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T16:01:04.428730335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:01:04.43224273Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T16:01:07.001626656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:01:07.004007619Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-09T16:01:07.490236166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:01:07.492976783Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:01:09.285539806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:01:09.288184919Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:01:09.805337414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:01:10.337255926Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-09T16:01:11.617994159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:01:11.624654652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:01:11.719852911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:01:11.723775955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T16:01:12.002182749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:01:13.013053486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:01:13.015933423Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:01:13.670160895Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T16:01:14.972122215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59240"
level=info ts=2026-05-09T16:01:17.002319095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T16:01:17.002327089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:01:17.00234542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:01:17.002355495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:01:17.018450229Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T16:01:17.018472421Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=info ts=2026-05-09T16:01:17.416065468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:01:17.421709199Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:01:20.336966662Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-09T16:01:20.632770359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:01:20.636430246Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:01:21.571321485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:01:21.574335807Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:01:21.929333091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:01:21.935195413Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:01:22.002302095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:01:22.030451609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:01:22.033287569Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:01:22.03330479Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:01:22.033325155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:01:22.035831295Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-09T16:01:23.416982899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:01:23.670200485Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T16:01:27.002313928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:01:27.003619351Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=info ts=2026-05-09T16:01:27.018540935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:01:27.022409456Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:01:29.520198362Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:01:30.35036547Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-09T16:01:30.350400544Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=4
level=warn ts=2026-05-09T16:01:32.002616211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:01:32.508399576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:01:32.512063076Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:01:33.101387602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:01:33.107974359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:01:33.107994399Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:01:33.108020529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:01:33.11425418Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:01:33.669770536Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-09T16:01:37.002179071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:01:37.003343651Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-09T16:01:37.031467353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:01:37.035203788Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:01:38.06621897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:01:38.070171046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:01:38.638729964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:01:38.642229269Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:01:38.833592377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:01:38.836561399Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:01:40.336609406Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-09T16:01:41.371492149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:01:41.375086642Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:01:42.00200037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:01:43.670012331Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T16:01:43.689980887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:01:43.693146203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T16:01:44.974782154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45360"
level=info ts=2026-05-09T16:01:46.907314915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:01:46.910490653Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T16:01:46.984660064Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:01:46.985053325Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:01:47.002405363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:01:47.002420318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T16:01:47.002413898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:01:47.002428159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:01:47.003613793Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-09T16:01:47.324625051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:01:47.327668462Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:01:47.93170477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:01:47.934774928Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:01:48.049264102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:01:48.05585339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:01:49.949890143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T16:01:49.958318858Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=debug ts=2026-05-09T16:01:50.336540565Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-09T16:01:51.030895603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:01:51.312617758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:01:51.315633869Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T16:01:52.002201403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:01:53.419110789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:01:53.670940963Z caller=checkpoint.go:372 msg="writing series" size="739 kB"
level=debug ts=2026-05-09T16:01:53.670963016Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-09T16:01:55.773312614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:01:55.778478951Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:01:57.001776452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:01:57.003319389Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=info ts=2026-05-09T16:01:58.261913186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:01:58.269785814Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:01:58.2698313Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:01:58.269859485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:01:58.276957716Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T16:02:00.336702083Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T16:02:01.216401618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:02:01.219256846Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T16:02:02.002090017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:03.095118517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:02:03.25946218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:02:03.26256617Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:02:03.669761203Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=warn ts=2026-05-09T16:02:07.002426971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:07.003976071Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T16:02:07.003991958Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T16:02:09.154354466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:02:09.158591173Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T16:02:09.502654476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:02:09.508338907Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:02:10.336925351Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-09T16:02:10.787425754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:02:10.790856583Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:02:12.00242344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:13.669953111Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T16:02:15.484076231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:02:15.493584647Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:02:17.002379099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:02:17.002419447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:02:17.00242762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:02:17.002430931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:17.004003401Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=info ts=2026-05-09T16:02:17.200124996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:02:17.203167973Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:02:17.309679657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:02:17.317759175Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:02:20.22271329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:02:20.226086144Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:02:20.336413722Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-09T16:02:22.002781257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:02:22.90246608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:02:22.905876286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T16:02:23.420494359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T16:02:23.66998152Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T16:02:27.002020725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:27.003126611Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-09T16:02:27.403156786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:02:27.40624726Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:02:30.336506282Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T16:02:31.736282568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:02:31.740106813Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T16:02:32.001786607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:02:32.528667184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:02:32.531310126Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:02:33.670168737Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=debug ts=2026-05-09T16:02:34.887028193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:02:35.951988011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:02:36.280106897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:02:36.28286423Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:02:36.664806498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:02:36.668971704Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:02:37.001784613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:37.004191343Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=debug ts=2026-05-09T16:02:37.004206298Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-05-09T16:02:37.778207555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:02:37.782051393Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:02:38.325173216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:02:38.328075686Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:02:40.337141764Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T16:02:40.337165253Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T16:02:40.945301928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:02:40.948431097Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T16:02:42.002444866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:02:43.091070578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:02:43.094149287Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T16:02:43.670428693Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-09T16:02:44.980963053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33026"
level=info ts=2026-05-09T16:02:45.782754969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T16:02:45.790407458Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
ts=2026-05-09T16:02:46.273365137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:37362"
level=info ts=2026-05-09T16:02:46.984693388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:02:46.985094538Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:02:47.002447022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:02:47.002463522Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:02:47.002471012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:02:47.002511403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:47.003975353Z caller=checkpoint.go:372 msg="writing series" size="694 kB"
level=debug ts=2026-05-09T16:02:50.337072231Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-09T16:02:50.756245883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:02:50.75932113Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:02:51.782350217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:02:51.788464269Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:02:51.880721187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:02:51.887158706Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:02:52.00205824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:02:53.422071391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:02:53.670477451Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T16:02:57.00206379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:02:57.003147715Z caller=checkpoint.go:372 msg="writing series" size="907 B"
level=info ts=2026-05-09T16:02:58.467097799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:02:58.470485399Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:02:59.691354294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:02:59.694676661Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:02:59.75071519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:02:59.754917952Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T16:03:00.336685991Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T16:03:01.000711577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:03:01.007871712Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:03:02.002659159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:03.669760858Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-09T16:03:03.669781396Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T16:03:04.196431509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:03:04.201900098Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:03:04.455585787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:03:05.786413437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:03:05.790530242Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T16:03:07.002291697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:07.006581123Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T16:03:07.006622943Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-09T16:03:07.61085081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:03:07.614080287Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:03:10.337152013Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-09T16:03:11.715113965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:03:11.718336057Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T16:03:12.002010375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:13.670588365Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-09T16:03:14.983938847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35256"
level=info ts=2026-05-09T16:03:15.648221255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="634 B" avg_comp="634 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:03:15.651403196Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:03:16.967862307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:03:16.974674421Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:03:17.002591783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:03:17.002601574Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:03:17.002621453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:03:17.002629333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:03:17.003751319Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T16:03:19.927337437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:03:19.930122073Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:03:20.336328179Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T16:03:20.509917666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:03:20.51328446Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T16:03:22.001997061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:03:23.423081209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:03:23.487849289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:03:23.492738934Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T16:03:23.670208222Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T16:03:25.454227132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:03:25.457514784Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:03:26.718012014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:03:26.720844684Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T16:03:27.001678341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:27.00397545Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-09T16:03:28.664332697Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:03:30.339869655Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-09T16:03:30.339897668Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=7
level=warn ts=2026-05-09T16:03:32.001914839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:03:33.074103774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47282"
level=info ts=2026-05-09T16:03:33.398446505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d77b00d39a40ef4 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-hral\"}"
level=error ts=2026-05-09T16:03:33.405361285Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d77b00d39a40ef4/19e0d3b65ad%3A19e0d4a78b3%3A783313a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-hral\"}"
level=error ts=2026-05-09T16:03:33.405382173Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:03:33.405406195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:03:33.407858381Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:03:33.670276544Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-09T16:03:35.068059024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:03:35.069739428Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T16:03:35.22331707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:03:35.229777796Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:03:35.245843729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:03:35.248431404Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:03:35.557210942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:03:35.562949239Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:03:37.001902142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:37.003159888Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T16:03:39.720555781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:03:39.72372493Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:03:40.162031618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:03:40.164879157Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T16:03:40.33646868Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-09T16:03:42.00248566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:03:43.443673875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:03:43.4472582Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:03:43.447281384Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:03:43.44730832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:03:43.450127846Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:03:43.66999244Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-09T16:03:46.277049434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:36252"
level=info ts=2026-05-09T16:03:46.591443878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:03:46.59507553Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:03:46.984519949Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:03:46.985366708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:03:47.001630942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:03:47.001658256Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:03:47.001666287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:03:47.001663518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:47.004021258Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-09T16:03:49.850717697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:03:49.856307519Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:03:50.337029914Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T16:03:51.126398313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:03:51.130637915Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T16:03:51.710242922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:03:51.713129022Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:03:52.0019181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:03:53.01356177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:03:53.016471642Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:03:53.078801028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T16:03:53.424795211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:03:53.670192321Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-09T16:03:53.670214599Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T16:03:55.929346686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:03:55.936088207Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:03:57.002378461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:03:57.00360639Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T16:04:00.336527193Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T16:04:00.579415999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:04:00.582608592Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T16:04:02.002328101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:04:02.930742828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:04:02.934975495Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-09T16:04:03.076545337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58094"
level=debug ts=2026-05-09T16:04:03.67059015Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-09T16:04:04.309439555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:04:04.312520371Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:04:07.002196904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:04:07.003301204Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-09T16:04:09.920165619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:04:09.923447178Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:04:10.336890339Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=debug ts=2026-05-09T16:04:10.880492583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T16:04:12.002449559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:04:12.902860527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:04:12.905976184Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:04:13.669934366Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T16:04:14.547136252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:04:14.550323044Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
ts=2026-05-09T16:04:16.278542375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58014"
level=info ts=2026-05-09T16:04:17.002004544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T16:04:17.00200918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:04:17.002029855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:04:17.002038032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:04:17.003635301Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-09T16:04:17.003649175Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=info ts=2026-05-09T16:04:18.176914798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:04:18.179926056Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:04:18.197000144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:04:18.200940915Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:04:20.213619668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:04:20.218256423Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:04:20.336497018Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-09T16:04:22.002364834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:04:22.931568202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:04:22.934454225Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:04:23.025982683Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:04:23.057997574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:04:23.061372183Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-09T16:04:23.42599281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:04:23.670391577Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T16:04:24.826562111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:04:24.832866843Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:04:27.002380828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:04:27.07171565Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-09T16:04:27.071745986Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-09T16:04:29.657579372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:04:29.670985416Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:04:30.337762416Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-09T16:04:30.671085996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:04:30.678950353Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:04:30.944428994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:04:30.962774351Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:04:32.001989727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:04:32.956951004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:04:32.967468576Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T16:04:33.67028899Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-09T16:04:33.715902623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:04:33.722678011Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:04:35.979249952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:04:35.990101374Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:04:37.001898792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:04:37.005719301Z caller=checkpoint.go:372 msg="writing series" size="902 kB"
level=debug ts=2026-05-09T16:04:37.005745661Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-09T16:04:40.337596942Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T16:04:40.337626012Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=warn ts=2026-05-09T16:04:42.002425426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:04:43.399153356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:04:43.402607292Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:04:43.670489796Z caller=checkpoint.go:372 msg="writing series" size="409 kB"
level=info ts=2026-05-09T16:04:44.432438222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:04:44.440578978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:04:45.562474375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:04:45.568750011Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:04:45.615066056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:04:45.619332206Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T16:04:46.984540311Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:04:46.985325257Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:04:47.001606102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T16:04:47.001611811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:04:47.001624418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:04:47.001632412Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:04:47.003824813Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-09T16:04:48.446910151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:04:48.450100349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:04:48.450361808Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:04:48.453292432Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:04:50.339438172Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T16:04:50.339463337Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=debug ts=2026-05-09T16:04:50.972600379Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T16:04:52.002280095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:04:53.42792477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:04:53.670956427Z caller=checkpoint.go:372 msg="writing series" size="860 kB"
level=warn ts=2026-05-09T16:04:57.002569427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:04:57.003643562Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T16:04:57.429801497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:04:57.433833815Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:05:00.336985423Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T16:05:02.002410768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:05:03.670358973Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=debug ts=2026-05-09T16:05:03.670379233Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-09T16:05:06.420189291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T16:05:07.002487997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:05:07.00361797Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T16:05:08.101676213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:05:08.107763895Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:05:08.216168365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:05:08.219719683Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:05:08.627719298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:05:08.632858437Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:05:08.632879847Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:05:08.632902718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:05:08.636025037Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:05:09.312440946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:05:09.315295649Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:05:10.336995329Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-09T16:05:10.466359579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:05:10.469286905Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T16:05:12.002294937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:12.03244032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:05:12.035188687Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:05:13.682000324Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-09T16:05:13.682022576Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=3
level=info ts=2026-05-09T16:05:14.160685676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:05:14.163786577Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
ts=2026-05-09T16:05:14.995952066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36436"
level=info ts=2026-05-09T16:05:17.001866632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T16:05:17.001887195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:05:17.001902807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:05:17.001912742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T16:05:17.010835197Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-09T16:05:17.01241243Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009541466s
level=info ts=2026-05-09T16:05:18.277998194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:05:18.281847794Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:05:19.399050319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:05:19.402931567Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:05:19.64349165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:05:19.650915379Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:05:22.002322086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:05:23.429963694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:05:23.851990209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:05:24.20399614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:05:24.207682438Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:05:26.234194745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:05:26.238148857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:05:26.679427204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:05:26.682625794Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T16:05:27.002585307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:29.065150878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:05:29.070113158Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T16:05:30.701305277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:05:30.704450821Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:05:30.704471663Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:05:30.704493754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:05:30.705520358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:05:30.706859612Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:05:30.706860964Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:05:32.002273371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:34.365532786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:05:34.370501029Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T16:05:37.002609732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:40.59118227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:05:40.600599232Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:05:42.002145078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:42.045281234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:05:42.048522804Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:05:43.106938647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:05:43.11244558Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:05:43.112469992Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:05:43.112506191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:05:43.117426434Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:05:43.980674695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:05:43.983499032Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T16:05:44.998776341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:40586"
level=info ts=2026-05-09T16:05:46.984619711Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:05:46.985035879Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:05:47.002382066Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T16:05:47.002384258Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:05:47.002401404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:05:47.002409307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:05:47.002407207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:47.002507817Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-09T16:05:47.002516462Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T16:05:47.002534833Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.851µs success=true
level=debug ts=2026-05-09T16:05:47.004153493Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T16:05:47.004174516Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=info ts=2026-05-09T16:05:47.504162568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:05:47.507112279Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:05:47.856337405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:05:48.174262761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:05:48.1773467Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:05:50.33672762Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T16:05:51.680947318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:05:51.684618592Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T16:05:52.002381405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:05:52.295847095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:05:52.298583359Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:05:52.380101853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:05:52.383519963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
ts=2026-05-09T16:05:53.431916827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:05:53.67008665Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=debug ts=2026-05-09T16:05:54.97669481Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:05:54.976701911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:05:54.980875801Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T16:05:57.002161426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:05:57.003287357Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-09T16:05:59.819854298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:05:59.824197383Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:05:59.824216575Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:05:59.824239481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:05:59.826735609Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:06:00.336146627Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T16:06:01.129343567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:06:01.134523225Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:06:02.002557647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:06:02.100137398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:06:02.103140635Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:06:02.260248719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:06:02.263122284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T16:06:03.086277769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46586"
level=debug ts=2026-05-09T16:06:03.670544702Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T16:06:05.36018842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:06:05.365594233Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:06:07.00247917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:07.003679408Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-09T16:06:07.253318777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:06:07.260742362Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:06:09.812257473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:06:09.815203872Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:06:10.336895432Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-09T16:06:11.152226741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:06:11.155972091Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T16:06:12.00210644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:13.669687718Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-09T16:06:13.873941007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:06:13.876911972Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T16:06:13.876932371Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:13.876953021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:06:13.879790214Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T16:06:13.884843701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:06:13.888014808Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:06:14.269938951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:06:14.272886878Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:06:15.005204177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:06:15.008619705Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:06:15.483972274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:06:15.486831232Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:06:15.866688538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:06:15.870933665Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T16:06:17.002546076Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:06:17.002581267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:06:17.002589439Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:06:17.00261755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:17.019716517Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T16:06:17.019736767Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=info ts=2026-05-09T16:06:18.05238579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:06:18.055431292Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:06:18.516732004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:06:18.519797902Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T16:06:20.33703421Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T16:06:22.002428845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:06:23.433778009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:06:23.670264836Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T16:06:25.633279603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:06:25.644855658Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:06:27.002450245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:27.004104518Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T16:06:28.331843865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:06:30.351832473Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-09T16:06:30.351858027Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=4
level=warn ts=2026-05-09T16:06:32.002557698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:06:32.029686452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:06:32.034912154Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:06:32.851690883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:06:32.854716016Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T16:06:33.669470272Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-09T16:06:33.969724479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:06:35.783296502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:06:35.786670651Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T16:06:35.786690104Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:35.786712409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:06:35.78941201Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:06:35.862530471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:06:35.865012831Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:06:35.865028543Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:36.087182042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:06:36.089891121Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T16:06:37.002449354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:37.003514102Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-09T16:06:39.18240137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:06:39.185459447Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:06:40.336832355Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-09T16:06:41.958492552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:06:41.961611285Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:06:41.961631877Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:41.961654865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:06:41.963926546Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T16:06:42.001864084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:06:43.620040582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:06:43.624032238Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T16:06:43.6705107Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T16:06:44.522768342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:06:44.525924658Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:06:44.525947683Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:44.525970152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:06:44.528375165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:06:45.725699299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:06:45.72962673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:06:46.984596579Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:06:46.984607293Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:06:47.001963858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:06:47.001982695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:06:47.001990535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:06:47.001994269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:47.003169245Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-09T16:06:47.535471132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:06:47.53849406Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:06:48.472353821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:06:48.475775752Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:06:49.214857047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:06:49.219111205Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T16:06:50.336700856Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=warn ts=2026-05-09T16:06:52.002395898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:06:52.304096778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:06:52.307521971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:06:52.307537137Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:52.3075576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:06:52.310532074Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T16:06:53.435662721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:06:53.494482984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:06:53.497466417Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T16:06:53.670292968Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-09T16:06:53.670316602Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-09T16:06:54.252238251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:06:54.255224577Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:06:54.58327005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:06:54.586225385Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:06:56.186629436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:06:56.190183822Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:06:56.855108251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:06:56.858071596Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:06:56.950458498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:06:56.955853406Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:06:57.001973496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:06:57.003353067Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=info ts=2026-05-09T16:06:57.383559258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:06:57.386257663Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:06:57.38627667Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:06:57.386296265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:06:57.388598238Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:06:58.137969703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:06:58.144597127Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T16:07:00.336736872Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T16:07:02.002453536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:07:02.805084125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:07:02.809176036Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:07:03.669689572Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-05-09T16:07:04.227597992Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:07:04.788124141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:07:04.791896306Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:07:06.477510053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:07:06.481268085Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:07:07.001617971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:07:07.003223847Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T16:07:07.003244743Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-09T16:07:08.920146305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:07:08.923365564Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:07:08.923385789Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:07:08.923406009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:07:08.926088854Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:07:09.413712334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:07:09.416429035Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:07:09.416450625Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:07:09.416471516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:07:09.418613529Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:07:09.661196298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:07:09.664125588Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:07:10.252109588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="795 kB" avg_comp="795 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T16:07:10.256643877Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0d340a74%3A19e0d59d6a6%3A77ff106f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-09T16:07:10.256664023Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:07:10.256688537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:07:10.260416743Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:07:10.337244676Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=warn ts=2026-05-09T16:07:12.001870632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:07:12.629926496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:07:12.636416655Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:07:13.149079348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:07:13.150756579Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T16:07:13.670176384Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T16:07:13.871181673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:07:13.874891117Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:07:16.40860313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:07:16.412078282Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:07:17.001597312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:07:17.0016266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:07:17.001635153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:07:17.001639923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:07:17.003219117Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=info ts=2026-05-09T16:07:17.988298171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:07:17.991542204Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:07:19.274982772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="58 MB" avg_comp="2.8 MB" total_uncomp="480 MB" avg_uncomp="23 MB" full=19 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:07:19.280561742Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:07:20.074713499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:07:20.077725122Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T16:07:20.336496641Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-09T16:07:22.001993284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:07:23.437393366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T16:07:23.669892843Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T16:07:27.002642601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:07:27.003636275Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-09T16:07:28.486762239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:07:28.490391308Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:07:28.948525297Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:07:29.221403213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:07:29.224954015Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:07:29.859761278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:07:29.864227457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:07:30.33659544Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T16:07:31.486846459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:07:31.490248182Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:07:31.490269712Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:07:31.490290818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:07:31.492788504Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:07:31.658275017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:07:31.66152334Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:07:32.002474039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:07:33.66967984Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-09T16:07:35.631291341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:07:35.636105224Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:07:35.821664758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:07:35.825159789Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T16:07:37.002204684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:07:37.003729213Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=debug ts=2026-05-09T16:07:37.003744483Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-05-09T16:07:37.906646687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:07:37.911341111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:07:38.337408404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:07:38.341557029Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:07:39.732495722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:07:39.73562231Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:07:40.33756421Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T16:07:40.337592745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T16:07:40.451940126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:07:40.456082678Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:07:40.622599141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:07:40.626520246Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:07:40.839198411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:07:40.849452429Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:07:42.002134501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:07:42.916757061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:07:42.919994051Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:07:43.669529427Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-09T16:07:44.053989854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:07:44.0571033Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:07:45.236012572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:07:45.242403235Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:07:45.242424127Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:07:45.242453375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:07:45.247635023Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:07:46.98451957Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:07:46.985010022Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:07:47.002381072Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:07:47.002400242Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T16:07:47.002401282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:07:47.002409419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:07:47.003778535Z caller=checkpoint.go:372 msg="writing series" size="600 kB"
level=debug ts=2026-05-09T16:07:50.337138681Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-09T16:07:52.002362231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:07:53.438939847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:07:53.670225294Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T16:07:54.832695212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:07:54.836496067Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T16:07:56.2402906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T16:07:57.00194135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:07:57.003024039Z caller=checkpoint.go:372 msg="writing series" size="907 B"
level=info ts=2026-05-09T16:07:57.311604451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:07:57.315412656Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T16:07:58.750189756Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:08:00.035392213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:08:00.039140857Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:08:00.336967555Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T16:08:01.164889174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:08:01.169575333Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T16:08:01.710291103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:08:01.713543949Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:08:02.002182736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:08:03.53194859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:08:03.535237101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:08:03.669845558Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-09T16:08:05.107713198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:08:05.11277742Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:08:05.254113733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:08:05.257282389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:08:05.355540169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:08:05.362478919Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:08:05.362500982Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:08:05.362527881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:08:05.369777246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:08:07.002192491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:08:07.006321055Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T16:08:07.006337891Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=8
level=info ts=2026-05-09T16:08:07.468376211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="504 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:08:07.475608353Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:08:10.337168253Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T16:08:12.002403392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:08:12.682822377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:08:12.685988007Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:08:13.670081833Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-09T16:08:14.77481983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:08:14.778141549Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T16:08:16.299355636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:51976"
level=warn ts=2026-05-09T16:08:17.001668413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:08:17.001667088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:08:17.001748586Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:08:17.001757589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:08:17.002842835Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T16:08:17.23893573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:08:17.242489169Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T16:08:17.871695882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:08:17.879295279Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:08:20.204401177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:08:20.208222745Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T16:08:20.336618639Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T16:08:20.489186494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:08:20.492478164Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T16:08:22.002330989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:08:23.35909297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:08:23.363054768Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:08:23.363075003Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:08:23.363096926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:08:23.365572144Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-09T16:08:23.440630446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T16:08:23.670120889Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T16:08:24.50764987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:08:24.510970219Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:08:26.782025193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:08:26.789870532Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:08:26.789899123Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:08:26.789935674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:08:26.798265252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:08:27.001773314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:08:27.003109623Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=info ts=2026-05-09T16:08:30.080213835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="634 B" avg_comp="634 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:08:30.083689096Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:08:30.340208145Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-09T16:08:30.340230521Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=7
level=debug ts=2026-05-09T16:08:30.347576621Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:08:30.347613616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="899 kB" avg_uncomp="899 kB" idle=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:08:30.351192709Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:08:30.942828983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:08:30.945965992Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:08:32.001755228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:08:33.669992342Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-09T16:08:34.652261126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:08:34.655561982Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:08:37.001927323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:08:37.003110213Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-09T16:08:37.155390807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:08:37.163785908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:08:38.376584622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:08:38.381258798Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:08:39.253598037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:08:39.256981448Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:08:40.327784077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:08:40.331752013Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T16:08:40.336830366Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=info ts=2026-05-09T16:08:40.855103258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:08:40.858515099Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=debug ts=2026-05-09T16:08:41.738876878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T16:08:42.002073468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:08:42.541464017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:08:42.548954331Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:08:42.951045113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:08:42.955335825Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:08:43.670054789Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-09T16:08:43.733074211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:08:43.738485608Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:08:46.984872396Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:08:46.985381717Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:08:47.001653893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:08:47.001674077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:08:47.001682542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:08:47.001713991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:08:47.003502885Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
level=info ts=2026-05-09T16:08:47.021401398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:08:47.029547491Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:08:50.337072667Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T16:08:51.501332802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:08:51.50491569Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T16:08:52.002210673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:08:52.42069426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:08:52.424005492Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T16:08:53.442461174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T16:08:53.670179121Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-09T16:08:53.67020209Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-09T16:08:55.208671973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:08:55.21469789Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T16:08:57.001782737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:08:57.003173168Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-09T16:08:57.124414265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="504 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:08:57.129540158Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:08:57.346859951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:08:57.349573823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:09:00.336985419Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-09T16:09:02.002511789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:09:02.930097695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:09:02.933259041Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T16:09:03.102164145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51010"
level=info ts=2026-05-09T16:09:03.249291263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:09:03.252206072Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:09:03.670318885Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-09T16:09:05.183961939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:09:05.445583647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:09:05.450037292Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T16:09:07.002067263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:09:07.003197798Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-09T16:09:07.019303141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:09:07.023363565Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:09:09.942271336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:09:09.947661821Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:09:10.337127617Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-09T16:09:11.590864855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:09:11.594616118Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:09:12.002430619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:09:13.222090852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:09:13.225596581Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:09:13.669495842Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-09T16:09:13.669523102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T16:09:14.565572569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:09:14.568851809Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
ts=2026-05-09T16:09:15.025069686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:48716"
level=info ts=2026-05-09T16:09:16.125360325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:09:16.129384727Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
ts=2026-05-09T16:09:16.304390154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55092"
level=warn ts=2026-05-09T16:09:17.002096812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:09:17.002130001Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:09:17.002148472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:09:17.002158591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:09:17.004070769Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=info ts=2026-05-09T16:09:17.6509532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:09:17.660003834Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T16:09:20.336530835Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-09T16:09:22.002167631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:09:23.446892091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:09:23.669965321Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T16:09:26.074519392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:09:26.260933146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:09:26.266092022Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:09:26.413959767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:09:26.418043174Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T16:09:27.002372082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:09:27.094097152Z caller=checkpoint.go:372 msg="writing series" size="64 MB"
level=debug ts=2026-05-09T16:09:27.094133592Z caller=checkpoint.go:384 msg="flushing series" totalSize="65 MB" series=4
level=info ts=2026-05-09T16:09:27.934502288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:09:27.95079089Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:09:27.973476233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:09:27.982678827Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:09:29.138748254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:09:29.162892404Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:09:29.248855068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:09:29.271514925Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:09:30.338695149Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-09T16:09:32.00199381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:09:33.104647577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42892"
level=debug ts=2026-05-09T16:09:33.670323918Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-09T16:09:37.003056873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:09:37.006095515Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T16:09:37.00612599Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-09T16:09:37.823255859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:09:37.853977488Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:09:38.023181461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:09:38.034492354Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:09:39.052621755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:09:39.067542008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:09:39.283889414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:09:39.293156873Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:09:40.287031258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:09:40.339155516Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T16:09:40.339199137Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=warn ts=2026-05-09T16:09:42.001914706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:09:42.551597312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:09:42.563216368Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:09:43.132073664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:09:43.149420716Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:09:43.670362451Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
ts=2026-05-09T16:09:45.029647443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33634"
level=info ts=2026-05-09T16:09:46.98617715Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:09:46.987788451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:09:47.001579469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:09:47.001620724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T16:09:47.001636735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:09:47.001636201Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:09:47.002804129Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=debug ts=2026-05-09T16:09:50.344225531Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-09T16:09:50.348031109Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=info ts=2026-05-09T16:09:50.611675344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:09:50.627124673Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T16:09:50.962140733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:09:50.971792809Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T16:09:52.001912849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:09:53.452234547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=debug ts=2026-05-09T16:09:53.671246205Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T16:09:53.671281978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T16:09:53.837535887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:09:53.847188998Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:09:54.569040635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:09:54.578764203Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:09:56.569949336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:09:56.585255812Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T16:09:57.001661194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:09:57.002775739Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T16:09:57.909344431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:09:57.917731858Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=debug ts=2026-05-09T16:10:00.336942753Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T16:10:02.00199049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:02.481313958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:10:02.489866292Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T16:10:03.107588072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55128"
level=info ts=2026-05-09T16:10:03.201449548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:10:03.209180389Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:10:03.670376079Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=info ts=2026-05-09T16:10:06.800057113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:10:06.829134409Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:10:07.002205583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:10:07.002843651Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-09T16:10:07.661284333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:10:07.664626374Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:10:07.664648883Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:10:07.664670911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:10:07.667165187Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:10:09.888376595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:10:09.894199805Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:10:10.028682044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:10:10.031733093Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:10:10.336605912Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-05-09T16:10:12.002278405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:12.957866118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:10:12.962746245Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:10:13.685241395Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-09T16:10:13.685265748Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=6
level=debug ts=2026-05-09T16:10:14.9032287Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:10:15.554998466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:10:15.5587093Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T16:10:16.309577612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53394"
level=info ts=2026-05-09T16:10:17.001581102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:10:17.001615133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:10:17.001626163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:10:17.001631498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:17.02564238Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-09T16:10:17.027388135Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.024710956s
level=info ts=2026-05-09T16:10:18.288660953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:10:18.291845278Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:10:20.445697043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:10:20.449053934Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:10:21.319468074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:10:21.32393743Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:10:21.486486011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:10:21.491073493Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=warn ts=2026-05-09T16:10:22.001888391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:10:23.45637607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:10:25.515259947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:10:25.518500445Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:10:25.801372505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:10:26.359737733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:10:26.36335229Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:10:26.363371751Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:10:26.363394348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:10:26.366314363Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:10:27.002180576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:28.621286863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:10:28.630442936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:10:32.002561235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:34.885588905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:10:34.888991664Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:10:35.416543808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:10:35.420328433Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:10:36.149340082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:10:36.154741302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:10:37.001728647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:37.062864165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:10:37.065786947Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:10:40.920573745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:10:40.925774567Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T16:10:41.021281543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:10:41.024071613Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T16:10:42.001995647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:43.135434521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:10:43.139768199Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:10:46.190929841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:10:46.910054135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:10:46.913536864Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:10:46.984590781Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:10:46.985001271Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:10:47.002325906Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T16:10:47.002332123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:47.002353125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:10:47.002366906Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:10:47.002374532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T16:10:47.002489306Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-09T16:10:47.002505097Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T16:10:47.002525897Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.344µs success=true
level=debug ts=2026-05-09T16:10:47.004965072Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T16:10:47.004975598Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=info ts=2026-05-09T16:10:47.09491518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:10:47.097805137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:10:50.256895398Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T16:10:51.032122822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:10:51.035002248Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T16:10:52.002438277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:10:53.458017057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:10:53.509122874Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-09T16:10:55.145391697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:10:55.152687683Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:10:55.285198195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:10:55.288273121Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:10:56.174714687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:10:56.180220642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:10:56.762558494Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-09T16:10:57.002380482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:10:59.301225111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:10:59.305899122Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:10:59.963306977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:10:59.966320196Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:11:00.014935361Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-09T16:11:01.252558969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:11:01.255639239Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=warn ts=2026-05-09T16:11:02.0023798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:11:02.933804982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:11:02.937547991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
ts=2026-05-09T16:11:03.11157547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37096"
level=debug ts=2026-05-09T16:11:03.26848761Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T16:11:05.454105845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:11:05.458498193Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:11:06.521691734Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-09T16:11:07.002525289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:11:09.774686761Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-09T16:11:09.992493386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:11:09.996167938Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:11:12.001859761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:11:13.027166916Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-09T16:11:13.71708803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:11:13.726917288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:11:13.985742673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:11:13.989086333Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:11:14.103310377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:11:14.106705373Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:11:14.304189451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:11:14.307109409Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:11:15.535476463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:11:15.539193764Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T16:11:16.293072467Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T16:11:16.293100243Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
ts=2026-05-09T16:11:16.313282115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48622"
level=debug ts=2026-05-09T16:11:16.509967347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:11:16.511539959Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T16:11:17.001727153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:11:17.001752832Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:11:17.001761186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:11:17.001804375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:11:19.406175943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:11:19.409111876Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:11:19.533459534Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-09T16:11:19.589881515Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:11:19.803361167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:11:19.805920697Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:11:21.976049682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:11:21.981503963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:11:22.001631612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:11:22.786907208Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
ts=2026-05-09T16:11:23.459854367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:11:26.039592167Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T16:11:27.001687509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:11:27.474936051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:11:27.48098615Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:11:29.2925813Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
level=info ts=2026-05-09T16:11:31.326927428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:11:31.329922919Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:11:31.537363484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:11:31.540116271Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T16:11:32.002196513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:11:32.568310429Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-09T16:11:32.568336747Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=5
level=info ts=2026-05-09T16:11:32.683204436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:11:32.686539487Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:11:35.798934072Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-09T16:11:37.002501021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:11:38.284007113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:11:38.288663574Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:11:39.05179769Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-09T16:11:42.00215844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:11:42.304700693Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=info ts=2026-05-09T16:11:43.740144523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:11:43.747062814Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:11:45.557765623Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
ts=2026-05-09T16:11:46.315160715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:51094"
level=info ts=2026-05-09T16:11:46.331548721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:11:46.33525099Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:11:46.984522876Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:11:46.985110324Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:11:47.002478659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:11:47.002494981Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:11:47.002502918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:11:47.002505024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:11:48.235786192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:11:48.239037063Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:11:48.810780771Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-09T16:11:48.826982515Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:11:49.204069764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:11:49.211734926Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:11:50.044999215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:11:50.04805759Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:11:50.262635245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:11:50.26576434Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T16:11:52.002208032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:11:52.063577749Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=info ts=2026-05-09T16:11:52.633247437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:11:52.638029313Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T16:11:53.460905902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:11:55.205640108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:11:55.3175719Z caller=checkpoint.go:372 msg="writing series" size="737 kB"
level=debug ts=2026-05-09T16:11:55.317594456Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=warn ts=2026-05-09T16:11:57.00225081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:11:57.427097083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:11:57.42972124Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:11:58.569930772Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=info ts=2026-05-09T16:11:59.673341552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:11:59.677089039Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:11:59.677108394Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:11:59.677130344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:11:59.680389719Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:12:00.827841218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:12:00.832268844Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:12:01.089830629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:12:01.093434976Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:12:01.09345618Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:01.093477929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:12:01.096314664Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:12:01.117465838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:12:01.119875644Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:12:01.644284846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:12:01.647466509Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:12:01.822073595Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T16:12:02.002644822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:12:03.023720465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:12:03.026667733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:12:03.026688389Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:03.026711315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:12:03.029122167Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T16:12:03.115744596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56518"
level=info ts=2026-05-09T16:12:03.455404096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:12:03.459241875Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T16:12:05.075978171Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-05-09T16:12:05.222636177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:12:05.229633804Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:12:07.002445249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:12:08.329454262Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-09T16:12:08.329477986Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-09T16:12:09.237631343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:12:09.240975199Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:12:10.69749629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:12:10.701660741Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:12:11.581550144Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-09T16:12:12.001623659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:12:14.834465254Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-09T16:12:15.403881301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:12:15.408663241Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:12:15.649240499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:12:15.652977368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:12:16.469273534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:12:16.472998937Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:12:17.001577402Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:12:17.001606303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:12:17.001613965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:12:17.001667875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:12:17.2948383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:12:18.088277662Z caller=checkpoint.go:372 msg="writing series" size="615 kB"
level=info ts=2026-05-09T16:12:20.322743638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:12:20.328540102Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:12:20.328561443Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:20.32860264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:12:20.334324007Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:12:21.094521857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:12:21.097563514Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:12:21.341221011Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=warn ts=2026-05-09T16:12:22.001746078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:12:22.447387516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:12:22.45047187Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T16:12:23.462988351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:12:24.593892529Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T16:12:27.002702715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:12:27.846853588Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=debug ts=2026-05-09T16:12:27.846877542Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T16:12:29.90193311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:12:29.905957564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:12:30.460461636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:12:30.463280487Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:12:31.100190244Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-09T16:12:32.002373161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:12:33.188516212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:12:33.193846418Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:12:33.437571945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:12:33.440737246Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:12:34.352446122Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-09T16:12:34.547003839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:12:34.5531455Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:12:36.54679316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:12:36.551866768Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T16:12:37.002475466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:12:37.60683101Z caller=checkpoint.go:372 msg="writing series" size="765 kB"
level=info ts=2026-05-09T16:12:38.752380827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:12:38.755873536Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:12:38.75589444Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:38.755917681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:12:38.759452166Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:12:40.457382146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T16:12:40.461178856Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0d342243%3A19e0d342273%3Ac65de7d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T16:12:40.461205525Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:40.461235972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:12:40.46460738Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T16:12:40.859758901Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T16:12:40.859781032Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
level=warn ts=2026-05-09T16:12:42.002225482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:12:43.326648855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="15 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:12:43.333703029Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:12:43.349720561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:12:43.35274922Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=debug ts=2026-05-09T16:12:44.112125519Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-09T16:12:46.32006512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48648"
level=info ts=2026-05-09T16:12:46.599363948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:12:46.602336661Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:12:46.984673489Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:12:46.985080637Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:12:47.00243233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:12:47.002457075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T16:12:47.002452526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:12:47.002467545Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T16:12:47.072783897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:12:47.075706414Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:12:47.075726182Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:47.075749382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:12:47.078175188Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:12:47.112150929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:12:47.115483447Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:12:47.2278587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:12:47.232459798Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T16:12:47.365070327Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=debug ts=2026-05-09T16:12:47.787022598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:12:48.479101177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:12:48.482703851Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:12:50.617909043Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-09T16:12:50.95174185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:12:50.954473895Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:12:50.954501599Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:50.954522741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:12:50.956728783Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:12:51.087562244Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T16:12:52.002277035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:12:53.465944698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:12:53.8706641Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T16:12:55.976056168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:12:55.979395641Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:12:56.453657714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:12:56.457025651Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-09T16:12:57.002011896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:12:57.123957936Z caller=checkpoint.go:372 msg="writing series" size="907 B"
level=info ts=2026-05-09T16:12:57.274504135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:12:57.277952556Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T16:12:57.277970417Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:12:57.277994516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:12:57.280931971Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:12:57.388538655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:12:57.392185608Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:12:58.006856493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:12:58.013099373Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:12:59.520261718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:12:59.523025495Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:13:00.376407298Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T16:13:00.573844591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="634 B" avg_comp="634 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:13:00.57816326Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=warn ts=2026-05-09T16:13:02.001654736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:13:03.119398332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39698"
level=debug ts=2026-05-09T16:13:03.629913285Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-09T16:13:03.629933485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T16:13:03.66191655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:13:03.668939968Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:13:04.583907354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:13:04.589327731Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:13:06.575563886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:13:06.579657151Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:13:06.885999929Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T16:13:06.886025955Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-09T16:13:07.002106091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:13:07.688452628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:13:07.69203485Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:13:08.13856839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:13:08.141431567Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:13:10.13600928Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-09T16:13:10.891349682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:13:10.894286622Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:13:11.214581292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:13:11.217437468Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:13:11.217457547Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:13:11.217479228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:13:11.219827596Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T16:13:12.002598966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:13.388643056Z caller=checkpoint.go:372 msg="writing series" size="335 kB"
level=info ts=2026-05-09T16:13:14.126334535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:13:14.129365116Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:13:14.129385434Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:13:14.129408792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:13:14.131732006Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:13:16.641855843Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T16:13:17.002604138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:13:17.002637681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:13:17.002647166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:13:17.00268616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:13:19.578620982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:13:19.583640343Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:13:19.894399425Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T16:13:22.002328618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:22.735624423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:13:23.147784353Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
ts=2026-05-09T16:13:23.468444242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:13:24.596388138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:13:24.600136422Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:13:26.088087298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:13:26.090829988Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:13:26.40095091Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-05-09T16:13:27.00257572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:29.659310205Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-09T16:13:29.659333515Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=7
level=info ts=2026-05-09T16:13:29.783474279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:13:29.787432859Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T16:13:30.078731439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:13:30.081718068Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:13:30.398678066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:13:30.402561252Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:13:31.354022701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:13:31.359052723Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:13:32.001861159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:13:32.227680342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:13:32.230865229Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:13:32.422746979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:13:32.4261639Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:13:32.426182864Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:13:32.426208786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:13:32.429007853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:13:32.614074397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:13:32.61817647Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:13:32.907143833Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
ts=2026-05-09T16:13:33.120914141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50918"
level=info ts=2026-05-09T16:13:33.46956025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:13:33.47239821Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:13:36.160211167Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=warn ts=2026-05-09T16:13:37.002541255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:39.412882809Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-05-09T16:13:40.527548688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:13:40.531979102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:13:40.763738984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:13:40.766792419Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:13:41.631297481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:13:41.634240674Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:13:41.634258762Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:13:41.634278073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:13:41.636558799Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T16:13:42.00165846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:42.665890179Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-09T16:13:42.786239313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:13:42.789259165Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:13:42.789278148Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:13:42.78930071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:13:42.791609077Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:13:43.085551615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:13:43.092878112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:13:43.344385709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:13:43.347952908Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-09T16:13:45.060088546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:60754"
level=debug ts=2026-05-09T16:13:45.918907406Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=info ts=2026-05-09T16:13:46.984518397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:13:46.984917003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:13:47.002215445Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T16:13:47.002219122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:47.002234563Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:13:47.002242613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:13:47.25768095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:13:47.530378158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:13:47.533783894Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:13:48.326849957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:13:48.326865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="764 kB" avg_comp="764 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" idle=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:13:48.342305518Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=info ts=2026-05-09T16:13:48.610214624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:13:48.617183529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:13:49.172155497Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T16:13:49.626774368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:13:49.630404137Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T16:13:52.002414393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:13:52.42540395Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-09T16:13:52.425426305Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
ts=2026-05-09T16:13:53.471224571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T16:13:53.66729354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:13:53.670779417Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:13:55.622410812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:13:55.625427065Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T16:13:55.677976743Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-09T16:13:56.647397845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:13:56.650453166Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T16:13:57.001943381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:13:58.765404303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:13:58.771563066Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:13:58.931028477Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T16:14:00.306617201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:14:00.309661321Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:14:00.309682017Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:14:00.309703033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:14:00.312162642Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:14:01.419499249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:14:01.422277757Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T16:14:02.002527935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:14:02.183652355Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-09T16:14:03.542005735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:14:03.546546197Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=info ts=2026-05-09T16:14:04.376799836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:14:04.379957452Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:14:04.818076764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:14:04.820986766Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:14:05.436912384Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-09T16:14:07.002398827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:14:08.690309281Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-09T16:14:09.641739253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:14:09.644895506Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:14:10.96567773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:14:10.969687193Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:14:11.943551286Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-09T16:14:11.943576622Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-09T16:14:12.002215974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:14:12.502650854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:14:12.506098595Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:14:15.007853197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:14:15.01138523Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T16:14:15.196495717Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=info ts=2026-05-09T16:14:15.2171365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:14:15.220915907Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T16:14:16.328104303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:59972"
level=info ts=2026-05-09T16:14:16.912075664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:14:16.91493178Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:14:17.001725206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:14:17.001753929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:14:17.001765028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:14:17.00176154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:14:17.284363057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:14:17.44384875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:14:17.446666142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:14:18.449574819Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-09T16:14:21.702170979Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T16:14:22.001919327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:14:23.474101001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T16:14:24.797299466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:14:24.80146617Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:14:24.801485978Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:14:24.801511225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:14:24.805207426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:14:25.049711737Z caller=checkpoint.go:372 msg="writing series" size="66 MB"
level=debug ts=2026-05-09T16:14:25.049773176Z caller=checkpoint.go:384 msg="flushing series" totalSize="67 MB" series=4
level=info ts=2026-05-09T16:14:25.535180212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:14:25.545610014Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:14:26.046544762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:14:26.056945129Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T16:14:27.001656079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:14:27.0831729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:14:27.091921734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:14:28.192269995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:14:28.199254899Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:14:28.208470495Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T16:14:29.190090638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:14:29.198860655Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:14:30.876799723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:14:30.890070759Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:14:30.890114577Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:14:30.890164475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:14:30.903798461Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:14:30.97439959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:14:30.985535208Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=debug ts=2026-05-09T16:14:31.461509275Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-09T16:14:32.001898738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:14:33.125934757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41824"
level=debug ts=2026-05-09T16:14:34.715687743Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T16:14:34.715715859Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-09T16:14:37.001654627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:14:37.527216898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:14:37.539235367Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:14:37.968589066Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T16:14:37.968614001Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-09T16:14:38.065482088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:14:38.08236751Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:14:40.002310728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:14:40.010384966Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:14:40.555251351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:14:41.220504014Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=warn ts=2026-05-09T16:14:42.002055027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:14:42.193889535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:14:42.211591485Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:14:42.211614598Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:14:42.21164331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:14:42.228021721Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:14:43.085508048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:14:43.097290433Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:14:43.153673331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:14:43.160851393Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:14:43.518248398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:14:43.527190688Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:14:44.473121261Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
ts=2026-05-09T16:14:45.066440971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:41542"
level=info ts=2026-05-09T16:14:46.357353967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:14:46.369137066Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:14:46.577556296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:14:46.585230867Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:14:46.9852973Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:14:46.986521453Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:14:47.002168415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:14:47.002198906Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:14:47.002209582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:14:47.002290705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:14:47.732130821Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-09T16:14:47.732168012Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=info ts=2026-05-09T16:14:48.347296844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:14:48.355906661Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:14:50.501529579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:14:50.980335244Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T16:14:50.980369768Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T16:14:52.002103505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:14:53.477905768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T16:14:53.594144218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:14:53.608101117Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:14:54.232008997Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T16:14:55.12667366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:14:55.144547818Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T16:14:57.001644493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:14:57.485787267Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T16:14:58.698981272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:14:58.713889339Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:14:59.069776619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:14:59.080347192Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:14:59.080389019Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:14:59.080420224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:14:59.088913114Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:15:00.738733469Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=warn ts=2026-05-09T16:15:02.001860921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:15:03.991516875Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-09T16:15:04.160323586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:15:04.167764969Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:15:04.59333967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:15:04.597377626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:15:05.994669675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:15:05.998180195Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:15:06.220825023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:15:06.224382566Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-09T16:15:07.002484801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:15:07.24446945Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-09T16:15:10.522244749Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-09T16:15:10.522298724Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=6
level=info ts=2026-05-09T16:15:11.299665927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:15:11.306365839Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T16:15:12.002601935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:12.622866589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:15:12.630491998Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:15:12.630516702Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:15:12.630537815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:15:12.635468169Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:15:12.809176068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:15:12.817431036Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=info ts=2026-05-09T16:15:13.284280717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:15:13.293867957Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:15:13.611397925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:15:13.624050093Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:15:13.750563361Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-09T16:15:14.11554382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:15:14.120252553Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T16:15:15.763109882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:15:15.764748638Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T16:15:16.615328321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:15:16.619798641Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T16:15:16.919841478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:15:16.92423737Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:15:17.00184647Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:15:17.001869102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T16:15:17.00185989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:17.001876892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:15:17.002949732Z caller=checkpoint.go:384 msg="flushing series" totalSize="215 kB" series=1
level=info ts=2026-05-09T16:15:17.012514342Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-09T16:15:17.014704713Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012000262s
level=info ts=2026-05-09T16:15:18.731201831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:15:18.739296574Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:15:18.739319714Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:15:18.739344477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:15:18.746759457Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:15:22.001673087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:15:23.483172658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=info ts=2026-05-09T16:15:24.306001678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:15:24.311142425Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:15:24.337224262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:15:24.343894368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:15:24.493583664Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:15:24.493625312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="936 kB" avg_comp="936 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:15:24.49773887Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=info ts=2026-05-09T16:15:24.637161195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:15:24.640213262Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:15:26.18172397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:15:26.184549444Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=warn ts=2026-05-09T16:15:27.001651119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:28.682171122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:15:28.685180257Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:15:30.936440168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:15:30.939491789Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T16:15:32.00164411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:32.552768982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:15:32.555698707Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T16:15:32.876929408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:15:32.879940238Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:15:32.99985694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:15:33.006255566Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:15:34.79692927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:15:34.799754779Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:15:34.886991417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:15:34.890267822Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:15:35.033581764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:15:35.037075706Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T16:15:37.002573205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:40.633323187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:15:40.638541016Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=warn ts=2026-05-09T16:15:42.002337611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:15:45.073750636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42460"
level=info ts=2026-05-09T16:15:46.985020739Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:15:46.985488499Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=debug ts=2026-05-09T16:15:46.986847987Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T16:15:46.98689352Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975936 n=1
level=debug ts=2026-05-09T16:15:46.986932645Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975936 n=1
level=info ts=2026-05-09T16:15:47.00177427Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T16:15:47.00180188Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:15:47.001832677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:15:47.001841525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:15:47.001868185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:47.0018907Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-09T16:15:47.001898799Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T16:15:47.001916823Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.388µs success=true
level=debug ts=2026-05-09T16:15:47.003869646Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T16:15:47.003879732Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-09T16:15:48.764841058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="529 MB" avg_uncomp="23 MB" full=21 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:15:48.771683299Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:15:50.255996678Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T16:15:51.670830104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:15:51.674157316Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:15:52.001768958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:52.306409109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:15:52.309506281Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:15:53.037003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:15:53.040887762Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-09T16:15:53.486228817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:15:53.508999997Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-09T16:15:54.117850616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:15:54.120933863Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:15:54.247338952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:15:54.255129489Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T16:15:55.580099346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:15:56.76169684Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-09T16:15:56.812145138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:15:56.815554256Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T16:15:57.001731375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:15:57.108624545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:15:57.112024094Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T16:15:59.895869096Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:15:59.895871472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="188 kB" avg_uncomp="188 kB" idle=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:15:59.899431229Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=debug ts=2026-05-09T16:16:00.014929751Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T16:16:02.002253048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:16:02.832550792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:16:02.83624497Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T16:16:03.267444106Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T16:16:03.797648722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:16:03.800756253Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:16:04.996766444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:16:05.001327195Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=info ts=2026-05-09T16:16:06.3895265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:16:06.394159603Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:16:06.520745301Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=warn ts=2026-05-09T16:16:07.002558612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:16:08.071948029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:16:08.074976142Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:16:08.585321288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:16:08.588514617Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:16:08.758748203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:16:08.76178018Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:16:08.761799711Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:16:08.761844641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:16:08.764290982Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:16:09.774274051Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T16:16:10.476457499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:16:10.481896017Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T16:16:12.001858018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:16:12.178328586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:16:12.184720827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:16:13.026746292Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T16:16:15.077085333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:50144"
level=info ts=2026-05-09T16:16:15.560566085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:16:15.563848709Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:16:15.634162195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:16:15.63725745Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:16:15.91900743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:16:15.921968679Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=debug ts=2026-05-09T16:16:16.294219662Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T16:16:16.294242052Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
level=info ts=2026-05-09T16:16:16.346536349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:16:16.349477279Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:16:17.001732658Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:16:17.001760989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:16:17.001769264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:16:17.001778482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:16:18.232545686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:16:18.235661531Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:16:18.822761387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:16:18.828729771Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=debug ts=2026-05-09T16:16:19.532795008Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-09T16:16:22.002328698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:16:22.78567429Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-09T16:16:22.880760317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:16:22.884329749Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T16:16:23.488280595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T16:16:24.512086334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:16:24.515118696Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:16:26.038899283Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-09T16:16:26.264314796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:16:26.269635928Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T16:16:27.001670651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:16:28.827576791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:16:29.292411591Z caller=checkpoint.go:372 msg="writing series" size="639 kB"
level=debug ts=2026-05-09T16:16:29.292435888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-09T16:16:30.839345684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:16:30.84268139Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T16:16:32.002423012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:16:32.56861838Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-09T16:16:32.568644148Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=1
ts=2026-05-09T16:16:33.135945774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46894"
level=info ts=2026-05-09T16:16:34.246872811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:16:34.25084544Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T16:16:35.798143127Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-09T16:16:37.002698162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:16:39.051349675Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-09T16:16:40.261160755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=22 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:16:40.268403068Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T16:16:42.001889588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:16:42.303976308Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-09T16:16:45.080520516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:42386"
level=debug ts=2026-05-09T16:16:45.557270867Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-09T16:16:46.466789213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:16:46.47065234Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=info ts=2026-05-09T16:16:46.717364307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:16:46.721431013Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T16:16:46.984590953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:16:46.984888377Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:16:47.002172613Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:16:47.002191208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T16:16:47.002183477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:16:47.002198977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T16:16:48.132586585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:16:48.137552094Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T16:16:48.504671395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:16:48.508354257Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:16:48.808234788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:16:48.809503005Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=error ts=2026-05-09T16:16:48.812127119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:16:51.560267273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:16:51.563624406Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:16:51.744174326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:16:51.747743133Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-09T16:16:52.002337639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:16:52.062882778Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=info ts=2026-05-09T16:16:52.124698535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:16:52.1285197Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T16:16:52.132124573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T16:16:52.761138637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:16:52.764340985Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:16:53.405719066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:16:53.413545588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T16:16:53.489982358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:16:54.363917754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:16:55.316380451Z caller=checkpoint.go:372 msg="writing series" size="643 kB"
level=debug ts=2026-05-09T16:16:55.316406827Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=info ts=2026-05-09T16:16:56.22972116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:16:56.232994275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T16:16:57.001674616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:16:57.745523355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:16:57.749078205Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:16:57.749097775Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:16:57.749118989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:16:57.751938092Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:16:58.569260462Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=info ts=2026-05-09T16:16:58.953235123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:16:58.956341421Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:16:59.829083265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:16:59.836404132Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:17:01.822358325Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T16:17:02.002668376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:02.146171585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:17:02.151149324Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=info ts=2026-05-09T16:17:02.822600154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:17:02.826436719Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T16:17:03.137667178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51326"
level=info ts=2026-05-09T16:17:03.447714211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:17:03.453640361Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:17:03.706436736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:17:03.709748652Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:17:05.074773118Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-09T16:17:07.001975838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:07.371841207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:17:07.375063068Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:17:07.475853789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:17:07.478688337Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:17:08.328655808Z caller=checkpoint.go:372 msg="writing series" size="774 kB"
level=debug ts=2026-05-09T16:17:08.328683798Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-09T16:17:09.902699215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:17:09.906055793Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:17:11.581468572Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T16:17:12.001883416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:17:14.834446443Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-09T16:17:17.00172653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:17.00174658Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:17:17.001765289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:17:17.001774063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T16:17:17.01392361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:17:17.017906529Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T16:17:17.303673112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:17:17.30815084Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=info ts=2026-05-09T16:17:17.349269732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:17:17.352725032Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T16:17:18.087066114Z caller=checkpoint.go:372 msg="writing series" size="615 kB"
level=debug ts=2026-05-09T16:17:21.340192105Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-09T16:17:22.00185002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:23.0884312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=22 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:17:23.095728486Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T16:17:23.491168526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:17:23.648161368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:17:23.651317281Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:17:24.593192961Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-09T16:17:26.412248205Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T16:17:27.001709219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:17:27.846093874Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=debug ts=2026-05-09T16:17:27.846116463Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-09T16:17:29.091199403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:17:29.095588164Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=debug ts=2026-05-09T16:17:31.099208209Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T16:17:31.445242186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:17:31.449625207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T16:17:32.001757718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:33.497959706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:17:33.502184869Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T16:17:34.352627014Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-09T16:17:37.002385241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:37.385348454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:17:37.388958406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T16:17:37.605779317Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-09T16:17:39.143726588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:17:39.146772987Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:17:40.670647608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:17:40.67389581Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T16:17:40.859059195Z caller=checkpoint.go:372 msg="writing series" size="990 kB"
level=debug ts=2026-05-09T16:17:40.859087931Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T16:17:41.385302731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:17:41.393849052Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T16:17:42.002710093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:42.116989169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:17:42.120287335Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=info ts=2026-05-09T16:17:42.317588279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:17:42.32410262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T16:17:42.400961466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T16:17:44.110965154Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-09T16:17:44.197384066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:17:44.201114005Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:17:44.619244881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:17:44.622154139Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-09T16:17:45.086846536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:60632"
level=info ts=2026-05-09T16:17:45.954295697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:17:45.958506987Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T16:17:46.984654466Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:17:46.985105928Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:17:47.002449138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:17:47.002467886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:17:47.002476838Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:17:47.002477167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:47.21525054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:17:47.219353411Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=debug ts=2026-05-09T16:17:47.364959458Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=info ts=2026-05-09T16:17:47.495857482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:17:47.499094752Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:17:49.255871263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:17:49.258980839Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:17:50.61684846Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-09T16:17:50.87451562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:17:50.881253789Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:17:51.273533343Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T16:17:52.002426565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:17:52.538675044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:17:52.54223664Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-09T16:17:53.493026826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wt7jz-155f7a68 10.129.0.95:7946"
level=debug ts=2026-05-09T16:17:53.869665133Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-09T16:17:54.952521813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:17:54.956504379Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:17:55.169149286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:17:55.172323142Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:17:56.887195591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:17:56.891245229Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:17:57.002566584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:17:57.123093916Z caller=checkpoint.go:372 msg="writing series" size="907 B"
level=info ts=2026-05-09T16:17:59.191310312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:17:59.195988247Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:17:59.833582494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:17:59.838089774Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=info ts=2026-05-09T16:18:00.041750249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:18:00.04536686Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:18:00.376334993Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T16:18:02.002072638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:18:03.629210529Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-09T16:18:04.59096532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="70 MB" avg_comp="2.8 MB" total_uncomp="578 MB" avg_uncomp="23 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:18:04.596986387Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:18:06.884202222Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T16:18:06.88422468Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=8
level=warn ts=2026-05-09T16:18:07.002252344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:18:10.135519598Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T16:18:12.001928225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:12.91970535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:18:12.923301032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:18:13.388114287Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-09T16:18:14.749713697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:18:14.753064598Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T16:18:15.094970401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38428"
level=debug ts=2026-05-09T16:18:16.640718723Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T16:18:17.002463092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:18:17.002490818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:18:17.002498483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:18:17.002493256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:18:19.894428303Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T16:18:21.352154493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:18:21.914936681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T16:18:22.002535414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:22.087177769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:18:22.090294662Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:18:23.14758944Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T16:18:23.202996648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:18:23.207620902Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-09T16:18:23.494694872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=debug ts=2026-05-09T16:18:26.399921895Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-05-09T16:18:27.001765974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:28.026875752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:18:28.032619842Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=info ts=2026-05-09T16:18:28.65220631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:18:28.655739033Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=debug ts=2026-05-09T16:18:29.657715796Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-09T16:18:29.657741535Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=7
level=info ts=2026-05-09T16:18:30.008934334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:18:30.016134719Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:18:31.110618205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:18:31.115111049Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:18:31.409027716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:18:31.413274331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:18:31.413294094Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:18:31.413317268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:18:31.417065505Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:18:31.748931012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:18:31.754551634Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:18:31.754573303Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:18:31.754606638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:18:31.757942865Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T16:18:32.001869174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:32.757310682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:18:32.76144059Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=debug ts=2026-05-09T16:18:32.906170292Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
ts=2026-05-09T16:18:33.145989672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38516"
level=debug ts=2026-05-09T16:18:36.159238641Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-09T16:18:36.872336754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:18:36.876633584Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-09T16:18:37.001627038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:37.467257307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:18:37.475567767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:18:37.960308809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:18:37.964768759Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:18:38.069348784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:18:38.076509152Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T16:18:38.214866877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:18:38.21888701Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-09T16:18:39.412517107Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-05-09T16:18:41.046935113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:18:41.049986037Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T16:18:42.002359341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:42.394693255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:18:42.400717693Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=info ts=2026-05-09T16:18:42.583167473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:18:42.587784324Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=debug ts=2026-05-09T16:18:42.665114355Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-09T16:18:42.878720735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="634 B" avg_comp="634 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:18:42.882478213Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:18:43.627886815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:18:43.634584822Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:18:43.936288537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:18:43.94016058Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T16:18:45.191008373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:18:45.197162464Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:18:45.290418479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:18:45.294117158Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:18:45.294139827Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:18:45.294163503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:18:45.296635915Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T16:18:45.918646962Z caller=checkpoint.go:372 msg="writing series" size="399 kB"
level=debug ts=2026-05-09T16:18:45.918682048Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-05-09T16:18:46.360745241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54928"
level=info ts=2026-05-09T16:18:46.984602618Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:18:46.984877724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:18:47.002167461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:18:47.002186907Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:18:47.002194814Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:18:47.002188967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:47.087316039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:18:47.090078425Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:18:47.103593542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:18:47.105959393Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T16:18:48.44161444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:18:49.171084947Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T16:18:52.00255202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:18:52.319489288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:18:52.322690417Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T16:18:52.424732816Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-09T16:18:53.496277491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T16:18:54.889168534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="70 MB" avg_comp="2.8 MB" total_uncomp="578 MB" avg_uncomp="23 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:18:54.895321106Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:18:54.895352215Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:18:54.895398621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="70 MB" avg_comp="2.8 MB" total_uncomp="578 MB" avg_uncomp="23 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:18:54.900152766Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T16:18:55.677452429Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=warn ts=2026-05-09T16:18:57.001790493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:18:58.930805649Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T16:19:01.9461501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:19:01.951407825Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:19:02.002614405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:19:02.183513544Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-09T16:19:05.321458381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:19:05.324780233Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:19:05.436177799Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-09T16:19:07.002318954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:08.201986222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:19:08.204943129Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=debug ts=2026-05-09T16:19:08.689461073Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-09T16:19:08.936945004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="70 MB" avg_comp="2.8 MB" total_uncomp="578 MB" avg_uncomp="23 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:19:08.943101593Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:19:09.767329183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:19:09.77035787Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:19:09.770379572Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:19:09.770400954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:19:09.772949187Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T16:19:11.74419112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="518 kB" avg_comp="518 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:19:11.748273798Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T16:19:11.942918224Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-09T16:19:11.942942473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-09T16:19:11.98777344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:19:11.991497315Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T16:19:12.00256388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:12.452448661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:19:12.454962426Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T16:19:14.699615054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T16:19:14.701341653Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T16:19:15.197050225Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
ts=2026-05-09T16:19:16.362595754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:43486"
level=info ts=2026-05-09T16:19:17.001678454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:19:17.00171763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:19:17.001726893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:19:17.001756343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:19:18.252618613Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:19:18.448243485Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-09T16:19:20.429861264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:19:20.432925929Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=debug ts=2026-05-09T16:19:21.701609608Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T16:19:22.002394627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:22.504639885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:19:22.508561856Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T16:19:22.772458512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:19:22.775293016Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T16:19:23.498367435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T16:19:24.881212685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:19:24.884295884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:19:24.884315118Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:19:24.884339307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:19:24.886955863Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:19:24.895020824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:19:24.897425777Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:19:24.89743775Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=debug ts=2026-05-09T16:19:25.064314636Z caller=checkpoint.go:372 msg="writing series" size="72 MB"
level=debug ts=2026-05-09T16:19:25.064347934Z caller=checkpoint.go:384 msg="flushing series" totalSize="73 MB" series=4
level=info ts=2026-05-09T16:19:25.184510464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:19:25.191624576Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=info ts=2026-05-09T16:19:26.902895594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:19:26.914133665Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T16:19:27.002250502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:27.442407217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:19:27.453180073Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=debug ts=2026-05-09T16:19:28.207384982Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-09T16:19:28.869515001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:19:28.881082728Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-09T16:19:28.881115406Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:19:28.881153408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:19:28.89455775Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:19:29.367134489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:19:29.375756971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T16:19:30.239077983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:19:30.257980775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:19:30.467538927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:19:30.475487112Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:19:30.475528675Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:19:30.475568959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:19:30.483114082Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:19:31.29905293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:19:31.307868259Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:19:31.30992241Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:19:31.309983475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:19:31.318254748Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T16:19:31.360759356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:19:31.371929935Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=debug ts=2026-05-09T16:19:31.461200601Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-05-09T16:19:31.540107417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="70 MB" avg_comp="2.8 MB" total_uncomp="578 MB" avg_uncomp="23 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:19:31.554235704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:19:31.586754979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:19:31.594318441Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T16:19:32.00194338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:34.040602653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:19:34.061122729Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T16:19:34.71558124Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T16:19:34.715609103Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-09T16:19:36.198896986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:19:36.207560682Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T16:19:37.001679668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:37.651037223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:19:37.660591721Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:19:37.660637095Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:19:37.660670621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:19:37.668001956Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:19:37.794884229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:19:37.806917446Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T16:19:37.968478521Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T16:19:37.968505398Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-09T16:19:38.131606862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:19:38.13959975Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T16:19:41.220015892Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=info ts=2026-05-09T16:19:41.549106918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T16:19:41.557239211Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0d340a74%3A19e0d43aba2%3A7e09eda8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T16:19:41.734095567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:19:41.745091658Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T16:19:41.977739956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:19:41.985868491Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=warn ts=2026-05-09T16:19:42.002059368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:42.397952172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:19:42.405896565Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T16:19:43.640947618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:19:43.649187029Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T16:19:44.472863562Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-09T16:19:45.986602296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:19:45.995332794Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:19:46.984671299Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T16:19:46.98525604Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.81:11211 count=1
level=info ts=2026-05-09T16:19:47.002255159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T16:19:47.002286774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:19:47.002298818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T16:19:47.002312164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:19:47.734186612Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-09T16:19:47.734232246Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=3
level=info ts=2026-05-09T16:19:50.013687218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:19:50.022052928Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T16:19:50.273841411Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T16:19:50.979559756Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T16:19:50.979603888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=warn ts=2026-05-09T16:19:52.00189826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:19:53.500543369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-23a0b45b 10.130.0.66:7946"
level=info ts=2026-05-09T16:19:53.745907368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6fec5a5ae1214dee immediate=false num_chunks=1 total_comp="517 kB" avg_comp="517 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"product-kubearchive-logging\"}"
level=error ts=2026-05-09T16:19:53.756673953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6fec5a5ae1214dee/19e0d33b4a0%3A19e0d5336ed%3A419fa753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive-logging\"}"
level=debug ts=2026-05-09T16:19:54.231291065Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T16:19:56.401064084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="132 kB" avg_uncomp="132 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T16:19:56.409873895Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0d33a75a%3A19e0d3428e4%3A2e72214c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T16:19:57.001647436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:19:57.125384188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:19:57.136777267Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T16:19:57.484849326Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T16:19:58.600915311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="38 kB" avg_comp="38 kB" total_uncomp="405 kB" avg_uncomp="405 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T16:19:58.611346468Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0d33a399%3A19e0d3428e3%3Ab9439cb4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T16:20:00.739114985Z caller=checkpoint.go:372 msg="writing series" size="297 kB"
level=info ts=2026-05-09T16:20:01.901894209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="536 kB" avg_comp="536 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T16:20:01.914212687Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0d33a07a%3A19e0d77d26e%3A1d80f17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T16:20:02.001953218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:20:02.044626469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b6a68de85ee118 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mibk\"}"
level=error ts=2026-05-09T16:20:02.055696715Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b6a68de85ee118/19e0d3b3e9d%3A19e0d4ed702%3A239add57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mibk\"}"
level=info ts=2026-05-09T16:20:02.892262987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T16:20:02.90312267Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0d340a74%3A19e0d3428e4%3Ad8b26a33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-09T16:20:03.152708315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44056"
level=debug ts=2026-05-09T16:20:03.991294687Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-09T16:20:04.888703585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T16:20:04.897575734Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0d339cfc%3A19e0d7fb58e%3A1d685803?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T16:20:05.976922148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9a4914ebc20f5a5 immediate=false num_chunks=1 total_comp="652 B" avg_comp="652 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-qjvr\"}"
level=error ts=2026-05-09T16:20:05.99121636Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e9a4914ebc20f5a5/19e0d39abdc%3A19e0d39cf06%3Adff2ccf2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-qjvr\"}"
level=debug ts=2026-05-09T16:20:06.754462026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T16:20:07.002518382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:20:07.244972734Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-09T16:20:08.356227405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="440 kB" avg_comp="440 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-09T16:20:08.367904286Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e0d339e49%3A19e0d5f0e96%3A162aa02c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-09T16:20:09.716285458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="895 kB" avg_comp="895 kB" total_uncomp="4.6 MB" avg_uncomp="4.6 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T16:20:09.726233474Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0d339cfd%3A19e0d77db02%3A9cf41b04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T16:20:10.521830649Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-09T16:20:10.521866867Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=6
level=warn ts=2026-05-09T16:20:12.001839033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:20:12.133489348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="27 MB" avg_uncomp="14 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T16:20:12.151060877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0d339f22%3A19e0d4dfe57%3A659c7cf0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T16:20:13.108358114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e19f2738d5dad773 immediate=false num_chunks=1 total_comp="937 kB" avg_comp="937 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"build-e2e-ydwi\"}"
level=error ts=2026-05-09T16:20:13.123284809Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e19f2738d5dad773/19e0d3b32e5%3A19e0d538125%3Aaf434ca5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ydwi\"}"
level=info ts=2026-05-09T16:20:13.175291499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=25cf6ac8039e0cec immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="899 kB" avg_uncomp="899 kB" full=1 labels="{stream=\"integration2-htzn\"}"
level=error ts=2026-05-09T16:20:13.183528817Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/25cf6ac8039e0cec/19e0d445399%3A19e0d4804ea%3A3253b772?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-htzn\"}"
level=debug ts=2026-05-09T16:20:13.750115315Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-09T16:20:14.219159464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="18 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T16:20:14.236898746Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0d33a311%3A19e0d4e87b6%3A7c312041?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T16:20:15.11088114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:44538"
level=debug ts=2026-05-09T16:20:16.9850066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T16:20:17.001990021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T16:20:17.002011155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T16:20:17.002026768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T16:20:17.002038979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T16:20:17.002069561Z caller=checkpoint.go:384 msg="flushing series" totalSize="215 kB" series=1
level=info ts=2026-05-09T16:20:17.00844584Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-09T16:20:17.011074442Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009021818s
level=info ts=2026-05-09T16:20:17.224773242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T16:20:17.228418865Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0d3a57bc%3A19e0d3ba3c5%3Af9d6484c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T16:20:18.724544834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="70 MB" avg_comp="2.8 MB" total_uncomp="578 MB" avg_uncomp="23 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T16:20:18.73023391Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0d33a2ac%3A19e0d3a411e%3A75d678d9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T16:20:20.69092674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T16:20:20.693833888Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0d342244%3A19e0d3428e3%3Aa26ce588?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T16:20:21.044998463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T16:20:21.047803467Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0d340a74%3A19e0d342245%3A9471cb94?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T16:20:21.352641355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:20:21.356730532Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:20:21.356751823Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:20:21.356775132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8e95bff64159c6d immediate=false num_chunks=1 total_comp="773 kB" avg_comp="773 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"build-e2e-ornb\"}"
level=error ts=2026-05-09T16:20:21.36017833Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8e95bff64159c6d/19e0d3b13a5%3A19e0d4d2c77%3Ac4d89657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ornb\"}"
level=warn ts=2026-05-09T16:20:22.00168324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T16:20:23.503208295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mm6fm-5c0f2449 10.128.0.79:7946"
level=info ts=2026-05-09T16:20:23.62055245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T16:20:23.623395755Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0d339fb4%3A19e0d3428e3%3Afcce6d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T16:20:24.307731783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T16:20:24.313347132Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0d339cde%3A19e0d666ea5%3A5a1ba675?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T16:20:26.817986827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:20:26.821754519Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:20:26.821774335Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:20:26.821799133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T16:20:26.824903912Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0d339d02%3A19e0d4e39fe%3Abdeb9dc2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T16:20:27.001851893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T16:20:30.567138128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:20:30.571213444Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:20:30.57123377Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:20:30.571254612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T16:20:30.573892001Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0d340a74%3A19e0d3455d9%3Ae6cbbe35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T16:20:31.151576263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="368 kB" avg_comp="368 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T16:20:31.155059585Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0d33a321%3A19e0d513b86%3A7169a552?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T16:20:31.609421425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:20:31.612372168Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:20:31.612391442Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T16:20:31.612413008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="579 B" avg_comp="579 B" total_uncomp="853 B" avg_uncomp="853 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T16:20:31.614887625Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0d340a74%3A19e0d340a75%3A93ea508c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T16:20:32.001628823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"