level=info ts=2026-05-07T14:56:36.766526949Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T14:56:36.766559807Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-07T14:56:36.766649041Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-07T14:56:36.76897876Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T14:56:36.769612497Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:56:36.769735788Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.64 inf=eth0
level=info ts=2026-05-07T14:56:36.770514308Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-b883f4df
ts=2026-05-07T14:56:36.770600466Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T14:56:36.770612228Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.64 advertisePort=7946
level=debug ts=2026-05-07T14:56:36.770652855Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.64 advertisePort=7946
level=info ts=2026-05-07T14:56:36.770780948Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-07T14:56:36.775454212Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T14:56:36.775478369Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-07T14:56:36.777040121Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
ts=2026-05-07T14:56:36.777075818Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T14:56:36.777082099Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T14:56:36.777096664Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:56:36.777118574Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.340898ms
level=error ts=2026-05-07T14:56:36.777132155Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T14:56:36.777152764Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-07T14:56:36.777720447Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-07T14:56:36.777733759Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-07T14:56:36.777732344Z
level=debug ts=2026-05-07T14:56:36.777744496Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-07T14:56:36.77780326Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-07T14:56:36.777856824Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-07T14:56:36.777869699Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-07T14:56:36.777868644Z
level=debug ts=2026-05-07T14:56:36.778051196Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-07T14:56:36.779298219Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:56:36.779379862Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.64 inf=eth0
ts=2026-05-07T14:56:36.779485285Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:56:36.779498947Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T14:56:36.793400993Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-07T14:56:36.793459382Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-05-07T14:56:36.793466993Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-07T14:56:36.793497205Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-07T14:56:36.793510089Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:56:36.793522202Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-07T14:56:36.793697361Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-05-07T14:56:36.793706005Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:56:36.793788049Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T14:56:36.793816171Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-07T14:56:36.793835613Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-07T14:56:36.79383772Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-07T14:56:36.793841024Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-07T14:56:36.793843129Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=info ts=2026-05-07T14:56:36.793845562Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-07T14:56:36.793846835Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-07T14:56:36.793850465Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-07T14:56:36.793882046Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T14:56:36.793910033Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-07T14:56:36.794027302Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:56:36.794035271Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-07T14:56:36.794038938Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-07T14:56:36.794042039Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-07T14:56:36.794045066Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-07T14:56:36.794047839Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-07T14:56:36.794325456Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-07T14:56:36.79433888Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T14:56:36.794390437Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=26.077µs success=true
level=info ts=2026-05-07T14:56:36.794403933Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-07T14:56:36.794460432Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=134.553µs errors=false
level=info ts=2026-05-07T14:56:36.794475383Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-07T14:56:36.794593713Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=270.735µs errors=false
level=info ts=2026-05-07T14:56:36.794601835Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-07T14:56:36.794606912Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=283.767µs
level=info ts=2026-05-07T14:56:36.794649015Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-07T14:56:36.794663932Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T14:56:36.794673913Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=8.910473027s
level=info ts=2026-05-07T14:56:36.794688569Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-07T14:56:36.794750913Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-07T14:56:36.794770459Z caller=loki.go:615 msg="Loki started" startup_time=85.916298ms
level=debug ts=2026-05-07T14:56:36.794780755Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-07T14:56:36.794786571Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-07T14:56:36.794821381Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-07T14:56:37.832190011Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:56:37.832225436Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:56:40.465857802Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:56:40.465880444Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:56:41.795420792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:56:46.794859374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:56:48.263743913Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:56:48.263774356Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:56:51.795109596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:56:53.492963213Z caller=logging.go:137 msg="GET /ready (503) 114.972µs"
level=warn ts=2026-05-07T14:56:56.795153811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:56:58.483865597Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
level=warn ts=2026-05-07T14:56:58.48388714Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused\n\n"
level=warn ts=2026-05-07T14:57:01.795772255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:57:03.491362468Z caller=logging.go:137 msg="GET /ready (503) 70.066µs"
level=info ts=2026-05-07T14:57:06.79521607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T14:57:06.795244216Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-07T14:57:06.795251724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:57:06.795266297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:11.795775832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:57:13.269099863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T14:57:16.795328366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:21.794882727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:26.795322574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:57:27.90025828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43098"
ts=2026-05-07T14:57:27.900311772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.64:7946"
ts=2026-05-07T14:57:27.903499115Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
ts=2026-05-07T14:57:27.905709841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.80:7946"
ts=2026-05-07T14:57:27.908029448Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused"
ts=2026-05-07T14:57:27.909132884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
ts=2026-05-07T14:57:27.91100313Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
ts=2026-05-07T14:57:27.912008168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.77:7946"
level=info ts=2026-05-07T14:57:27.912933443Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=51.135774075s
level=debug ts=2026-05-07T14:57:31.619997551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:57:31.795690276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:57:35.217515286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T14:57:36.777113532Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:57:36.777109436Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T14:57:36.795444266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:57:36.795471432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:57:36.795482984Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:57:36.795480388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:57:39.071016351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T14:57:41.795855312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:46.795454976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:57:48.913892267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:48272"
level=debug ts=2026-05-07T14:57:51.07723818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:57:51.795537617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:56.795700376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:01.795041167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:58:05.220129192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T14:58:06.795676531Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T14:58:06.795677308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:58:06.795711904Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:58:06.795725149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T14:58:07.038058262Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T14:58:07.038105917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:58:07.047431574Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T14:58:08.327979664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:58:11.795653516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:16.795282182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:21.795187198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:58:25.87892919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:58:25.885417582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T14:58:26.795162044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:58:28.918958095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:58:31.795891429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:58:32.381083729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T14:58:35.222169582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T14:58:36.777205586Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:58:36.777767489Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T14:58:36.795140878Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:58:36.795179662Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T14:58:36.79518192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:58:36.795196301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:58:41.795023296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:58:45.751599017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44880"
level=warn ts=2026-05-07T14:58:46.795229902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:58:50.973431998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:58:50.981202482Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T14:58:51.795685162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:56.795664915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:59:00.617026043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T14:59:00.618968722Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:59:01.7955149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:59:04.710295674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46878"
ts=2026-05-07T14:59:05.224554384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T14:59:06.275234473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T14:59:06.794845888Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:59:06.79488387Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:59:06.794896015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:59:06.794999145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:11.795033795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:16.795026497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:21.795008585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:26.795650883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:31.795048037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:59:32.153317813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:59:32.161968469Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T14:59:33.718818648Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T14:59:35.226715736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T14:59:36.777119701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:59:36.777125089Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=warn ts=2026-05-07T14:59:36.795507119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:59:36.795500994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:59:36.795533875Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:59:36.795544415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T14:59:38.749218276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:59:41.795083249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:46.794910956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:51.795579516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:56.795166967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:00:01.79529542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:03.21706788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:00:05.228621542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:00:06.795461309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:00:06.795495978Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:00:06.795507787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:00:06.795517173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:00:11.795822545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:00:16.497993917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:00:16.509322991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:00:16.79571841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:18.922917744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51056"
level=warn ts=2026-05-07T15:00:21.795163347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:00:26.795500681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:30.774943295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:00:31.795123733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:35.231304315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:00:35.570902621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T15:00:36.777317684Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:00:36.777382042Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-07T15:00:36.778737652Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:00:36.778795772Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975739 n=1
level=debug ts=2026-05-07T15:00:36.778857294Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975739 n=1
level=info ts=2026-05-07T15:00:36.794759122Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:00:36.794788949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:00:36.794798765Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:00:36.794836532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:00:41.795836738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:45.758279188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50768"
level=warn ts=2026-05-07T15:00:46.79540545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:00:51.79540668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:56.686171918Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:00:56.795245697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:01:01.39960765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:01:01.412705285Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:01:01.795043926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:01:05.23379999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:01:06.795459221Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:01:06.795487706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:01:06.795495961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:01:06.795505164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:01:11.79543733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:01:16.795684007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:01:21.795890894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:22.308166724Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:01:26.795289696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:01:31.795312857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:01:35.236016718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:01:36.776784429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:01:36.777834231Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:01:36.795221412Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:01:36.795242393Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:01:36.795250709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:01:36.795238159Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T15:01:36.795255429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:01:36.795386338Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-07T15:01:36.795394883Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:01:36.795409214Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.664µs success=true
level=debug ts=2026-05-07T15:01:36.796372238Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T15:01:36.796382678Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-07T15:01:41.150877228Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-07T15:01:41.795759078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:45.505854606Z caller=checkpoint.go:372 msg="writing series" size="479 kB"
ts=2026-05-07T15:01:45.761625013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41036"
level=warn ts=2026-05-07T15:01:46.795740131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:49.860148677Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-07T15:01:51.209950896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:01:51.216609735Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:01:51.795561936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:54.030816625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T15:01:54.215384953Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:01:55.301956569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:01:56.795837629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:58.57050679Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=warn ts=2026-05-07T15:02:01.795379736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:02.925408266Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-07T15:02:04.720952938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51006"
ts=2026-05-07T15:02:05.237845376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:02:06.79534429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:02:06.795374178Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:02:06.795383133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:02:06.795402426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:07.280440394Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-07T15:02:07.280471378Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-07T15:02:11.634363546Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-07T15:02:11.795297261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:15.989975737Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-07T15:02:16.795534153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:02:18.929151454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:37666"
level=debug ts=2026-05-07T15:02:20.344581569Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:02:21.795503525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:24.69951436Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-07T15:02:26.795362987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:27.2060503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:02:29.054007643Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:02:31.795120526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:33.409017227Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-07T15:02:35.240190659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:02:36.777492718Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:02:36.778552379Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:02:36.794754888Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:02:36.794781157Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:02:36.794789111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:02:36.794798801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:37.763735973Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:02:41.795158664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:42.119040017Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=info ts=2026-05-07T15:02:44.391625822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:02:44.397985862Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:02:45.76498041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50870"
level=debug ts=2026-05-07T15:02:46.473454601Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-07T15:02:46.794820256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:50.828387573Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:02:51.795330745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:53.529506992Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:02:55.183774803Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=warn ts=2026-05-07T15:02:56.795695206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:57.126673244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T15:02:59.538512264Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-07T15:03:01.795732714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:03.893274269Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=debug ts=2026-05-07T15:03:03.893297983Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-05-07T15:03:05.24239553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:03:06.795375444Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:03:06.795402267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:03:06.795416855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:03:06.795429443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:08.247879395Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:03:11.795275956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:12.60266757Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-07T15:03:15.767112881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38398"
level=warn ts=2026-05-07T15:03:16.795715158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:16.957078711Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-05-07T15:03:21.311985514Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-07T15:03:21.795700589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:25.666651526Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:03:26.79518546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:28.356065557Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:03:30.021703085Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-07T15:03:31.795030762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:34.376502091Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-07T15:03:34.726468811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42962"
ts=2026-05-07T15:03:35.24430715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:03:36.777154199Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:03:36.777590939Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:03:36.79496274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:03:36.794999563Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:03:36.795010325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:03:36.795021727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:38.731331815Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=warn ts=2026-05-07T15:03:41.795707158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:03:41.815670817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:03:41.823393805Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:03:43.086508943Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-07T15:03:46.794901516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:47.441524015Z caller=checkpoint.go:372 msg="writing series" size="489 kB"
ts=2026-05-07T15:03:48.934918354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:60192"
level=debug ts=2026-05-07T15:03:50.138837752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T15:03:50.14047756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:03:51.795096588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:51.796141837Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:03:56.151147861Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:03:56.795250589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:59.386519607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:04:00.505821208Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:04:01.795958903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:04:04.728708932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:52168"
level=debug ts=2026-05-07T15:04:04.860734674Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
ts=2026-05-07T15:04:05.246668695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:04:06.794780576Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:04:06.794821001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:04:06.794843621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:04:06.794884634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:09.215444991Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-07T15:04:09.215465933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=15
level=warn ts=2026-05-07T15:04:11.795664002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:13.569977488Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-07T15:04:16.795360256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:17.924742492Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:04:21.795272283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:22.279708838Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T15:04:25.692692725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:04:25.704041555Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:04:25.704068437Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:04:25.704094878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:04:25.71019974Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:04:26.635039583Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-07T15:04:26.794904243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:30.98995042Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-07T15:04:31.028048967Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:04:31.795245219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:04:35.248929664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:04:35.345059501Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-07T15:04:36.777467926Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:04:36.777913795Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:04:36.795275148Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:04:36.795297912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:04:36.795305864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:04:36.79531889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:04:39.554432464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:04:39.562845962Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:04:39.69950258Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-07T15:04:41.79503741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:44.054040163Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-07T15:04:46.794906585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:47.739585813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:04:48.408723404Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-07T15:04:48.408747461Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-05-07T15:04:48.939057226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49306"
level=warn ts=2026-05-07T15:04:51.795136049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:52.76357443Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-07T15:04:56.795836009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:57.118866218Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T15:05:01.473770068Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-07T15:05:01.795803773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:05:04.345537206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:05:04.354894817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:05:04.73182325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:58724"
ts=2026-05-07T15:05:05.251402222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:05:05.759895009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:05:05.828373815Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-07T15:05:06.795658372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:05:06.795688161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:05:06.795697732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:05:06.795752577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:10.182945617Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T15:05:11.795103782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:14.538019381Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-05-07T15:05:15.772209472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34582"
level=warn ts=2026-05-07T15:05:16.79486121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:18.89317143Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-07T15:05:21.795886551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:23.247369052Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-07T15:05:26.795738647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:27.602926388Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T15:05:31.795407888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:31.957782362Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-05-07T15:05:35.253413583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:05:36.312777045Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-07T15:05:36.77777527Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:05:36.778526977Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:05:36.794754436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:05:36.794780295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:05:36.794789339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:05:36.794846275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:37.335426129Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:05:40.560308478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:05:40.667584955Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:05:41.795085811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:45.025787313Z caller=checkpoint.go:372 msg="writing series" size="5.4 MB"
level=debug ts=2026-05-07T15:05:45.0258128Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.8 MB" series=13
level=warn ts=2026-05-07T15:05:46.795656001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:05:48.943184428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51868"
level=debug ts=2026-05-07T15:05:49.376996152Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-07T15:05:51.505217657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:05:51.512971674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:05:51.795024917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:53.731710551Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-07T15:05:56.794865575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:58.086105951Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-07T15:06:01.795752022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:02.441725623Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:06:04.735670469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33768"
ts=2026-05-07T15:06:05.255528904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:06:06.794755769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:06:06.794786357Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:06:06.794794316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:06:06.794799541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:06.795812841Z caller=checkpoint.go:384 msg="flushing series" totalSize="310 kB" series=4
level=info ts=2026-05-07T15:06:06.800615267Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-07T15:06:06.804164032Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008624231s
level=debug ts=2026-05-07T15:06:08.004675294Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:06:11.795083593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:06:16.795472164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:21.008387361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:06:21.79503085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:06:26.795563682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:06:31.79513812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:34.807018576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:06:35.258244354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:06:36.777663003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:06:36.778753945Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:06:36.795030787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:06:36.7950556Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:06:36.79506437Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:06:36.795067193Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T15:06:36.795131767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:06:36.795218707Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-07T15:06:36.795227888Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:06:36.795245063Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.794µs success=true
level=debug ts=2026-05-07T15:06:36.796133021Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:06:36.796142736Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=debug ts=2026-05-07T15:06:41.151405109Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=warn ts=2026-05-07T15:06:41.795783888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:45.506327182Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-07T15:06:46.795192337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:49.860844212Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-07T15:06:50.827938646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:06:50.838555115Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:06:51.795441094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:54.215391069Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T15:06:56.795846053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:58.570111887Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=warn ts=2026-05-07T15:07:01.795090079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:02.925328589Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-07T15:07:05.26010811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:07:06.795275631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:07:06.795302044Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:07:06.795309929Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:07:06.795321123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:07.279779578Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-07T15:07:08.284259535Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:07:11.634370547Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:07:11.795054144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:07:15.778235413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47300"
level=debug ts=2026-05-07T15:07:15.989263963Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-07T15:07:15.989284685Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-07T15:07:16.795774914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:20.344708321Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-07T15:07:21.784295793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:07:21.795336462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:24.699236242Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-07T15:07:26.794809551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:29.054236686Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:07:31.795243254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:07:32.411334249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:07:32.420306053Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:07:33.409174439Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
ts=2026-05-07T15:07:35.262516015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:07:36.777218202Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:07:36.777224676Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:07:36.795606788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:07:36.79562877Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:07:36.795651848Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:07:36.795668788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:37.76330019Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-07T15:07:38.249985317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:07:41.794854631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:42.118308526Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=debug ts=2026-05-07T15:07:46.473578736Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-07T15:07:46.794822764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:50.827752849Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:07:51.795681001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:55.183121331Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-07T15:07:56.795213177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:59.537545453Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=warn ts=2026-05-07T15:08:01.795543545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:03.892445164Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
ts=2026-05-07T15:08:05.264471854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:08:06.794951405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:08:06.794980586Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:08:06.794988203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:08:06.794993131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:06.856171563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:08:08.247358039Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-07T15:08:11.795769216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:12.60294926Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-07T15:08:16.795115506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:16.957478416Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T15:08:16.985017307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:08:16.998445614Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:08:21.312352227Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=debug ts=2026-05-07T15:08:21.312373162Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=15
level=warn ts=2026-05-07T15:08:21.795310975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:23.671786686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:08:23.673442656Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:08:25.667374183Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:08:26.795843706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:30.02127251Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-07T15:08:31.795004875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:34.376923279Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-07T15:08:35.26723314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:08:36.777345977Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:08:36.778335801Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:08:36.794755374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:08:36.794784464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:08:36.794795877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:08:36.794878376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:38.731873246Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=warn ts=2026-05-07T15:08:41.795044779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:42.351267081Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:08:43.085826682Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-07T15:08:46.795532886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:47.441532749Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=debug ts=2026-05-07T15:08:51.795852794Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:08:51.795862417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:56.151237272Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:08:56.795455983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:00.505709264Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=warn ts=2026-05-07T15:09:01.795178337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:09:04.303657221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:09:04.312321441Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:09:04.860464416Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
ts=2026-05-07T15:09:05.269834751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:09:06.794843483Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:09:06.794869008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:09:06.794877865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:09:06.794882625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:09.215603202Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=debug ts=2026-05-07T15:09:09.21562457Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-07T15:09:11.795295329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:13.570342137Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-07T15:09:13.720181724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:09:14.130804209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:09:15.784790361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59418"
level=warn ts=2026-05-07T15:09:16.795180493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:17.925015645Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
ts=2026-05-07T15:09:18.954908897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:45788"
level=warn ts=2026-05-07T15:09:21.794928569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:22.279448542Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:09:26.634663685Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-07T15:09:26.795468411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:30.990122044Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-07T15:09:31.79542065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:09:34.746787655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35818"
ts=2026-05-07T15:09:35.271483454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:09:35.344251156Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-07T15:09:36.777248109Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:09:36.778321152Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:09:36.795779937Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:09:36.795807852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:09:36.795815738Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:09:36.795835703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:39.699298253Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-07T15:09:41.795691735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:42.131684483Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:09:44.054378673Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-07T15:09:44.054401229Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
ts=2026-05-07T15:09:45.786848842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44844"
level=warn ts=2026-05-07T15:09:46.795370629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:48.409381176Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-07T15:09:49.005915928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:09:49.016681149Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:09:51.795030923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:52.764110899Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-07T15:09:54.998233026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:09:56.795600909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:57.118882474Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T15:10:01.4735776Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-07T15:10:01.795671978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:10:05.273948089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:10:05.82806679Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-07T15:10:06.795386517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:10:06.795382343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:10:06.795412663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:10:06.795421487Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:10:10.183498228Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-07T15:10:11.450820976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:10:11.795201002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:14.538137276Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-07T15:10:15.789110486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33678"
level=warn ts=2026-05-07T15:10:16.795624088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:18.893352613Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-07T15:10:21.795116322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:23.247626603Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-07T15:10:26.794916009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:27.602897449Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T15:10:31.795185893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:31.957561583Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-07T15:10:35.275730707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:10:36.311780991Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-07T15:10:36.777802401Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:10:36.778769143Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:10:36.795187935Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:10:36.795212735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:10:36.79522657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:10:36.795280252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:40.058834676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:10:40.082070364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:10:40.089745528Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:10:40.0897708Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:10:40.089796117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:10:40.096499633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:10:40.667250449Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:10:41.795746761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:45.02545146Z caller=checkpoint.go:372 msg="writing series" size="6.3 MB"
level=debug ts=2026-05-07T15:10:45.02547356Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.8 MB" series=14
level=warn ts=2026-05-07T15:10:46.79494712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:48.549241734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:10:48.959143069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34368"
level=debug ts=2026-05-07T15:10:49.376891707Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-07T15:10:51.795032524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:53.731908407Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-07T15:10:56.794829931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:58.086331481Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-07T15:10:59.266343083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:10:59.27418049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:11:01.795296569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:02.441450231Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:11:04.75085561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38208"
ts=2026-05-07T15:11:05.278378282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:11:06.795216957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:11:06.795247778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:11:06.795256123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:11:06.795279272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:06.796324007Z caller=checkpoint.go:384 msg="flushing series" totalSize="316 kB" series=4
level=info ts=2026-05-07T15:11:06.801716391Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-07T15:11:06.80327249Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007889344s
level=warn ts=2026-05-07T15:11:11.795140175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:12.768733614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:11:16.795610508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:11:21.79576923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:26.782670221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:11:26.795814528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:11:31.795707548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:11:34.753341359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42208"
ts=2026-05-07T15:11:35.280362581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:11:35.858012059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:11:35.866618078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:11:36.777248628Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:11:36.777684492Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:11:36.795212488Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:11:36.795244189Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:11:36.795247612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:36.795262056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:11:36.79526965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:11:36.795348472Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-07T15:11:36.795355952Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:11:36.795373952Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.789µs success=true
level=debug ts=2026-05-07T15:11:36.796297473Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T15:11:36.79630641Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-07T15:11:41.150866701Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=warn ts=2026-05-07T15:11:41.794892829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:44.152032298Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:11:45.505847588Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
ts=2026-05-07T15:11:45.793834526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37946"
level=warn ts=2026-05-07T15:11:46.795262407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:49.860857199Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:11:51.795650078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:54.215583126Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T15:11:56.795016442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:58.57072759Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-07T15:12:01.795216406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:02.924838152Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-07T15:12:04.755519196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41038"
ts=2026-05-07T15:12:05.282502048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:12:06.795097221Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:12:06.795131847Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:12:06.795142303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:12:06.795147531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:07.280450445Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-07T15:12:07.280473107Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-07T15:12:11.635068024Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-07T15:12:11.795776812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:14.830842208Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:12:15.989461271Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-07T15:12:16.795717798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:19.792492629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:12:19.794068859Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:12:20.344085099Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:12:21.795089766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:24.699295968Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T15:12:26.230145948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:12:26.237982473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:12:26.79523204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:29.054132328Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:12:31.795046963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:33.409442582Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
ts=2026-05-07T15:12:34.757940443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51220"
ts=2026-05-07T15:12:35.285028775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:12:36.777396646Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:12:36.77791878Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:12:36.795456095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:12:36.795479791Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:12:36.795488074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:12:36.795519161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:37.763576255Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:12:41.795342076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:42.118518519Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=debug ts=2026-05-07T15:12:42.934304795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:12:46.473784864Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-07T15:12:46.795060478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:12:48.966023462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59952"
level=debug ts=2026-05-07T15:12:50.828415316Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:12:51.795106242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:55.182808105Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-07T15:12:56.795790218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:59.538161347Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=warn ts=2026-05-07T15:13:01.79509602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:03.893293428Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
ts=2026-05-07T15:13:04.760081398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46800"
ts=2026-05-07T15:13:05.287555903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:13:06.795185368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:13:06.795211159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:13:06.795218866Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:13:06.795220707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:08.247966968Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-07T15:13:08.247989337Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=14
level=warn ts=2026-05-07T15:13:11.795287299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:12.602349662Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-07T15:13:13.978163406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:13:15.004830606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:13:15.798970075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34150"
level=warn ts=2026-05-07T15:13:16.795340521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:16.957749866Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-05-07T15:13:18.968229449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52222"
level=info ts=2026-05-07T15:13:18.985959178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:13:18.992796136Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:13:21.312303689Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-07T15:13:21.795847781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:25.666933267Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:13:26.79542734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:30.022395799Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-07T15:13:31.79507736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:34.376897675Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-07T15:13:35.289535013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:13:36.777783853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:13:36.778603046Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:13:36.795020206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:13:36.795046987Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:13:36.795057306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:13:36.795071114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:38.731249833Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-07T15:13:41.560309239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:13:41.795403207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:43.086550178Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-07T15:13:45.800715801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57404"
level=warn ts=2026-05-07T15:13:46.795497346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:47.441917285Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=warn ts=2026-05-07T15:13:51.795721733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:51.795757365Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:13:56.151445112Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:13:56.795164207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:00.505965136Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:14:01.691355616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:14:01.795804871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:04.86037564Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-07T15:14:04.860399652Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=info ts=2026-05-07T15:14:05.065966384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:14:05.073051052Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:14:05.291480456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:14:06.795275557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:14:06.795304061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:14:06.795312242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:14:06.795342349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:09.215052922Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=warn ts=2026-05-07T15:14:11.795390591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:13.570237754Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-07T15:14:16.457991599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:14:16.795492086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:17.925429569Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:14:21.795255221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:22.27986678Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:14:26.635290585Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-07T15:14:26.795364129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:30.98996971Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-05-07T15:14:31.795045054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:14:35.293599414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:14:35.344603735Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T15:14:35.344624979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-07T15:14:36.777301869Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:14:36.777867724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:14:36.795345352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:14:36.795379326Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:14:36.795389002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:14:36.795390753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:39.583670517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:14:39.699008211Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-07T15:14:41.795670573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:44.054488175Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-07T15:14:46.795168617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:48.409029521Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
ts=2026-05-07T15:14:48.973420129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38998"
level=debug ts=2026-05-07T15:14:49.823051013Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:14:49.823118592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:14:49.831034575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:14:51.795174889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:52.764029965Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-07T15:14:56.795075359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:57.118182171Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T15:15:01.47356078Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-07T15:15:01.795706205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:15:03.998605382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:15:04.007263358Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:15:05.295509175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:15:05.828415218Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T15:15:06.795685303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:15:06.795694434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:06.795712967Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:15:06.795721139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:15:07.808450364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:15:07.814786915Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:15:10.18284061Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-07T15:15:11.795185983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:14.278877882Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:15:14.538564716Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-07T15:15:16.794806324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:18.893075047Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=warn ts=2026-05-07T15:15:21.795318854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:23.248139925Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-07T15:15:26.795213099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:27.602447147Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-07T15:15:27.795698053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:15:31.795054825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:31.957422169Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-07T15:15:35.297939954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:15:36.311991215Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-07T15:15:36.776990927Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:15:36.778058167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:15:36.779322638Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:15:36.779367952Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975740 n=1
level=debug ts=2026-05-07T15:15:36.779425876Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975740 n=1
level=info ts=2026-05-07T15:15:36.795571104Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:15:36.795594585Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:15:36.795602965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:15:36.795589951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:40.666672219Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:15:41.7949959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:15:42.68007949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:15:42.687059519Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:15:44.951049628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:15:45.029473331Z caller=checkpoint.go:372 msg="writing series" size="7.4 MB"
level=debug ts=2026-05-07T15:15:45.029495629Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.3 MB" series=16
level=warn ts=2026-05-07T15:15:46.794943173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:49.376898Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=warn ts=2026-05-07T15:15:51.795079476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:53.731427655Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-07T15:15:55.891704907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:15:55.906824268Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:15:56.795344596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:58.086842546Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:16:01.794914069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:02.441370884Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:16:04.769832248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36228"
ts=2026-05-07T15:16:05.299997007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:16:06.79540836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:16:06.795434171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:16:06.795441523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:16:06.795447345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:06.796516857Z caller=checkpoint.go:384 msg="flushing series" totalSize="323 kB" series=4
level=info ts=2026-05-07T15:16:06.807697292Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-07T15:16:06.809223084Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.0137198s
level=debug ts=2026-05-07T15:16:11.502737743Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:16:11.795693662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:16:16.795826396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:16:18.978806691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59550"
level=warn ts=2026-05-07T15:16:21.79571001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:24.784166242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:16:24.786444763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:16:26.79558681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:16:31.795852362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:16:35.302165534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:16:35.802087587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:16:36.777369669Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:16:36.778401876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:16:36.794849331Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T15:16:36.79486719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:16:36.794896553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:16:36.794918916Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:16:36.794929384Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:16:36.795010297Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-07T15:16:36.795018304Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:16:36.795037207Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.328µs success=true
level=debug ts=2026-05-07T15:16:36.796129707Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:16:36.796140199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-07T15:16:37.91720512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:16:37.925986215Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:16:38.686916528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:16:38.691656866Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:16:41.151137978Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=warn ts=2026-05-07T15:16:41.795254187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:45.505484034Z caller=checkpoint.go:372 msg="writing series" size="584 kB"
level=warn ts=2026-05-07T15:16:46.79503859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:49.860105594Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:16:51.795649519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:54.215503957Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T15:16:56.795336681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:58.569888569Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-07T15:16:58.569919373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-07T15:17:01.112755584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:17:01.795693211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:02.924497176Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-07T15:17:04.99250318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:17:04.992544762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="316 kB" avg_comp="316 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:17:04.997784622Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:17:05.30439481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:17:06.795555085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:17:06.795559228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:06.79558217Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:17:06.795590477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:17:07.279859017Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-07T15:17:11.634331469Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:17:11.795199285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:17:15.810893774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33756"
level=debug ts=2026-05-07T15:17:15.989903387Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-07T15:17:16.795352737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:17:17.131258217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:17:17.139554831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:17:18.98204421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56946"
level=debug ts=2026-05-07T15:17:20.344429334Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:17:21.795106561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:24.699274552Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-07T15:17:26.795727908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:29.054364935Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:17:31.795086177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:17:32.565986305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:17:32.57300543Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:17:33.408369156Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-07T15:17:33.423477039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:17:33.430802267Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:17:33.430829605Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:17:33.430854071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:17:33.437494403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:17:35.015823843Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:17:35.306221582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:17:36.777058089Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:17:36.778101959Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:17:36.795696954Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:17:36.795706743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:36.795722893Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:17:36.795731234Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:17:37.763003196Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:17:41.795650605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:42.118575785Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=debug ts=2026-05-07T15:17:43.661661119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:17:44.334667477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:17:44.340037148Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:17:46.10533549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:17:46.112448524Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:17:46.472802812Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T15:17:46.794852177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:17:48.984570772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59612"
level=debug ts=2026-05-07T15:17:50.827937317Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:17:51.79505494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:55.18326341Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=warn ts=2026-05-07T15:17:56.795796145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:59.538131933Z caller=checkpoint.go:372 msg="writing series" size="411 kB"
level=debug ts=2026-05-07T15:17:59.53815983Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=14
level=warn ts=2026-05-07T15:18:01.795190951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:03.892363358Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-07T15:18:05.308343734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:18:06.795585095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:18:06.795624122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:18:06.795654822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:18:06.795704762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:07.092282738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:18:08.24745171Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-07T15:18:11.795027281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:12.601944255Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-07T15:18:16.79581022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:16.957259811Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-07T15:18:17.319314572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:18:17.328516497Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:18:18.981282356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
ts=2026-05-07T15:18:18.987035856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39566"
level=error ts=2026-05-07T15:18:18.988429179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:18:21.312103122Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-07T15:18:21.795273231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:25.666475555Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:18:26.795123707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:18:28.619906466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:18:28.62397549Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:18:30.021710357Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-07T15:18:31.795128076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:32.53868851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:18:34.3760986Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-07T15:18:35.310325287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:18:36.777537402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:18:36.777961906Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:18:36.795623567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:18:36.79566624Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:18:36.795675949Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:18:36.795676001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:38.731552073Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-07T15:18:39.831079683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:18:41.795393068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:43.085992379Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-07T15:18:46.795347554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:47.441535272Z caller=checkpoint.go:372 msg="writing series" size="592 kB"
level=debug ts=2026-05-07T15:18:47.441556936Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=11
level=warn ts=2026-05-07T15:18:51.795321194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:51.79538344Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-07T15:18:56.150941564Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:18:56.795410669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:00.50567668Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-07T15:19:01.662776346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:19:01.67458817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:19:01.795080241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:04.860042943Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
ts=2026-05-07T15:19:05.312651933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:19:06.124356185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:19:06.124414372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:19:06.132766458Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:19:06.795045833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:19:06.7950799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:19:06.795087968Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:19:06.795097619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:09.216069429Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=info ts=2026-05-07T15:19:10.420062735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:19:10.424378755Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:19:11.795223199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:19:13.4150331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:19:13.420187014Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:19:13.569996028Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
ts=2026-05-07T15:19:15.817873322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40394"
level=info ts=2026-05-07T15:19:16.553379939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:19:16.560896654Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:19:16.795242659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:17.924935571Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
ts=2026-05-07T15:19:18.991357106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:47720"
level=warn ts=2026-05-07T15:19:21.79588928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:22.279463481Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:19:26.634730967Z caller=checkpoint.go:372 msg="writing series" size="363 kB"
level=debug ts=2026-05-07T15:19:26.634754045Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-07T15:19:26.795568587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:30.989254932Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-07T15:19:31.716807945Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:19:31.795409481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:32.526568102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:19:35.314620604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:19:35.344017782Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=debug ts=2026-05-07T15:19:36.776911063Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:19:36.777954033Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:19:36.795533942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:19:36.795561669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:19:36.795570362Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:19:36.795597102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:39.69932368Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-07T15:19:41.795614022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:44.053901433Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=warn ts=2026-05-07T15:19:46.79571638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:48.408744553Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-07T15:19:49.538856507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:19:49.548396454Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:19:51.795748077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:52.763808158Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-07T15:19:52.763831764Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-07T15:19:53.795281589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:19:53.802062865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:19:56.795469084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:57.11852026Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:19:58.269132627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:19:58.274246531Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:19:58.399682458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:19:58.403731611Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:20:01.473481406Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-07T15:20:01.79539303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:20:05.316966922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:20:05.82787813Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=info ts=2026-05-07T15:20:06.794914899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:20:06.794943736Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:20:06.794951869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:20:06.794965062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:07.524534789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:20:10.183146767Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-07T15:20:11.795796218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:14.538078639Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-07T15:20:16.795246306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:18.15210008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:20:18.154175463Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:20:18.892658919Z caller=checkpoint.go:372 msg="writing series" size="405 kB"
level=warn ts=2026-05-07T15:20:21.79533544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:23.247895465Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-07T15:20:26.795739771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:27.602397779Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T15:20:31.795600314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:31.957004045Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-07T15:20:33.563909828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:20:33.57420884Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:20:35.319810631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:20:36.312005081Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-07T15:20:36.777172948Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:20:36.777189685Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:20:36.795685018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:20:36.795713411Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:20:36.7957228Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:20:36.795743783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:40.666375934Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-07T15:20:41.004902198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:20:41.795445066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:20:43.224982653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:20:43.233394262Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:20:45.026729728Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-05-07T15:20:45.026756886Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.2 MB" series=12
level=warn ts=2026-05-07T15:20:46.795487233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:49.376620308Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-07T15:20:51.795134119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:53.731458784Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-07T15:20:55.815266442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:20:55.82444631Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:20:56.576658455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:20:56.582097382Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:20:56.795621676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:58.086367199Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-05-07T15:21:01.795174709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:02.441168212Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:21:05.322496208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=warn ts=2026-05-07T15:21:06.795387752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:06.795402083Z caller=checkpoint.go:384 msg="flushing series" totalSize="197 kB" series=4
level=info ts=2026-05-07T15:21:06.795397028Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:21:06.795426196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:21:06.795436756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:21:06.798418174Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-07T15:21:06.800381761Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005185478s
level=debug ts=2026-05-07T15:21:07.876341146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:21:11.795356076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:15.367238786Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:21:16.795126005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:21:21.795036957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:21:26.795677273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:21:31.795192651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:21:32.685278154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:21:32.696032186Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:21:34.786534281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35830"
ts=2026-05-07T15:21:35.32454812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:21:36.777115254Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:21:36.777171887Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:21:36.794671593Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:21:36.794821102Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-07T15:21:36.794833517Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:21:36.79485403Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.851µs success=true
level=warn ts=2026-05-07T15:21:36.794889938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:21:36.794873404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:21:36.794923635Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:21:36.794935497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:21:36.795942892Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:21:36.795953637Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T15:21:39.675137932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:21:39.682645297Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:21:40.302804086Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=warn ts=2026-05-07T15:21:41.795600368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:43.816998279Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-07T15:21:43.8170257Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=info ts=2026-05-07T15:21:44.79875863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:21:44.803122768Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:21:45.825888728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60620"
level=warn ts=2026-05-07T15:21:46.795009229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:47.314810144Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-07T15:21:48.06405777Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:21:50.301971397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:21:50.308240316Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:21:50.308269107Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:21:50.308293759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:21:50.312840626Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:21:50.822096093Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=warn ts=2026-05-07T15:21:51.795756885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:54.328462691Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=warn ts=2026-05-07T15:21:56.794994143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:57.834773327Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-07T15:22:01.341389861Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:22:01.79523868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:22:04.789146143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45026"
level=debug ts=2026-05-07T15:22:04.84776016Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-07T15:22:04.847793898Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-07T15:22:05.326531134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:22:05.453674832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:22:06.795111184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:22:06.795150094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:22:06.795139628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:22:06.795159528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:22:08.354434227Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-07T15:22:09.25054372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:22:09.256133076Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:22:11.79590109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:11.861127567Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-07T15:22:15.367245583Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:22:16.795557474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:18.874070405Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-07T15:22:20.455039163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:22:20.468440318Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:22:21.795574018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:22.04103271Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:22:22.380527454Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:22:25.88614971Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-07T15:22:26.795074846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:29.393090333Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:22:31.795686726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:22:32.345328496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:22:32.355854254Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:22:32.900041667Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-05-07T15:22:34.791234007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45578"
ts=2026-05-07T15:22:35.329091474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:22:36.406145628Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-05-07T15:22:36.77714568Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:22:36.777551139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:22:36.795156759Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:22:36.79517837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:36.795189921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:22:36.795199166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:22:39.913061555Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:22:41.795185901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:43.418684085Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-07T15:22:43.77857729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:22:43.786617583Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:22:44.571021566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:22:44.576003638Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:22:45.829427856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54284"
level=warn ts=2026-05-07T15:22:46.79545201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:46.926267558Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=debug ts=2026-05-07T15:22:46.926289649Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=12
level=debug ts=2026-05-07T15:22:49.829028174Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:22:50.431762382Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T15:22:51.794860398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:53.938255868Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T15:22:56.795364353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:57.444596551Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-07T15:23:00.95142445Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-07T15:23:01.260586527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:23:01.795590021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:04.458448843Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
ts=2026-05-07T15:23:05.331264658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:23:06.795320738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:23:06.795349268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:23:06.795357598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:23:06.795372381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:07.965065066Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=debug ts=2026-05-07T15:23:11.471573353Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=info ts=2026-05-07T15:23:11.529616778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:23:11.540794438Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:23:11.794883184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:14.977901896Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-07T15:23:15.832213172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47182"
level=debug ts=2026-05-07T15:23:16.209039295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:23:16.493941709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:23:16.502801461Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:23:16.795164374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:18.484004349Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T15:23:21.795357736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:21.99060462Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:23:25.496558608Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-07T15:23:26.795044764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:23:27.996022769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:23:28.003814744Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:23:29.003946404Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-07T15:23:31.795289026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:32.510394326Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
ts=2026-05-07T15:23:35.333403023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:23:36.016669177Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-07T15:23:36.777211587Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:23:36.777964332Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:23:36.795469778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:23:36.79550377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:23:36.795514914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:23:36.795508132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:39.523109208Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-07T15:23:39.52314209Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-05-07T15:23:41.794979228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:43.029781131Z caller=checkpoint.go:372 msg="writing series" size="692 kB"
level=info ts=2026-05-07T15:23:44.432920769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:44.438730917Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:44.438755297Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:23:44.438780383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:44.443040635Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:23:45.834946255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53966"
level=debug ts=2026-05-07T15:23:46.536251944Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=warn ts=2026-05-07T15:23:46.795570375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:50.042725468Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:23:51.795772138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:51.913086432Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:23:53.548894211Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-07T15:23:56.033711787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:56.047104879Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:23:56.795048969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:57.058901927Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-05-07T15:23:57.058931186Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-07T15:23:57.662823939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:23:57.670873148Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:23:57.67089795Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:23:57.67092335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:23:57.678477524Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:24:00.561833166Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=debug ts=2026-05-07T15:24:01.230552598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:24:01.795953374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:04.069007585Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-07T15:24:04.609251059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:24:04.619844864Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:24:05.335672571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:24:06.794887242Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:24:06.794928479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:24:06.794939532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:24:06.795082502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:07.575308586Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:24:11.081228865Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-07T15:24:11.795229803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:14.58800978Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-07T15:24:14.778610385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:24:14.790253825Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:24:16.795450053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:24:17.799533243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:24:17.804762937Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:24:18.09451927Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:24:21.600726997Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-07T15:24:21.795586544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:24:23.933505415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:24:23.938303818Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:24:24.230219216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:24:25.1072495Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-07T15:24:26.795505583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:28.613933958Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-07T15:24:31.794868164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:32.120304854Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-07T15:24:32.120325212Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
ts=2026-05-07T15:24:34.799046772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46076"
ts=2026-05-07T15:24:35.341274419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:24:35.627312906Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:24:36.777232768Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:24:36.778287521Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:24:36.795780458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:24:36.795783816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:36.795808756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:24:36.795819248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:24:38.005612313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:24:38.015981502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:24:39.133764592Z caller=checkpoint.go:372 msg="writing series" size="389 kB"
level=debug ts=2026-05-07T15:24:41.225871235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:24:41.227812345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:24:41.79500735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:42.639793892Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-07T15:24:46.146010203Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-07T15:24:46.795332924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:49.65291375Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T15:24:49.948893933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:24:49.958689975Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:24:51.795331444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:53.158994663Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:24:56.665583281Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-07T15:24:56.79543902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:00.109368447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:25:00.172731905Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:25:01.795467467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:03.678866017Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
ts=2026-05-07T15:25:05.344243272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:25:06.794772026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:25:06.794807404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:25:06.794800584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:25:06.794818523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:25:07.185253492Z caller=checkpoint.go:372 msg="writing series" size="603 kB"
level=debug ts=2026-05-07T15:25:07.185275246Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-07T15:25:10.385500526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:25:10.393569207Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:25:10.483774938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:25:10.488860354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:25:10.691428412Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=warn ts=2026-05-07T15:25:11.795581122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:14.198360484Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
ts=2026-05-07T15:25:15.841055194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38422"
level=warn ts=2026-05-07T15:25:16.795755294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:17.704770393Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:25:19.904514129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:25:21.211692142Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:25:21.795581921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:24.718284163Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-07T15:25:26.650366436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:25:26.659655063Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:25:26.794809222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:28.223893226Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=debug ts=2026-05-07T15:25:31.730748738Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:25:31.795469764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:34.608909752Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:25:34.803553293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:52350"
level=debug ts=2026-05-07T15:25:35.237087288Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:25:35.345903889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:25:36.777244073Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:25:36.777875806Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:25:36.795411897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:25:36.795439434Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:25:36.795448519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:25:36.79551573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:38.748998129Z caller=checkpoint.go:372 msg="writing series" size="8.1 MB"
level=debug ts=2026-05-07T15:25:38.749025317Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.8 MB" series=9
level=warn ts=2026-05-07T15:25:41.795397634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:42.250689125Z caller=checkpoint.go:372 msg="writing series" size="497 kB"
level=debug ts=2026-05-07T15:25:45.757045821Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-07T15:25:46.795434087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:25:46.877755738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:25:46.886453542Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:25:46.886479721Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:25:46.88650923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:25:46.89319263Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:25:49.263728275Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=warn ts=2026-05-07T15:25:51.795277426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:25:51.854390983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:25:51.859587246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:25:52.769658118Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-07T15:25:56.275693121Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-07T15:25:56.795695853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:59.782709337Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-07T15:26:01.109188267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:26:01.126697029Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:26:01.301415799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:26:01.305705383Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:26:01.795847399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:03.289229186Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:26:05.348264013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:26:05.53181492Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:26:06.795052106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:26:06.795061963Z caller=checkpoint.go:384 msg="flushing series" totalSize="966 kB" series=7
level=debug ts=2026-05-07T15:26:06.795087172Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:26:06.795098601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:26:06.795155899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:26:06.923099049Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-07T15:26:06.933546896Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.138509753s
level=debug ts=2026-05-07T15:26:09.483651255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:26:10.869688451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:26:10.8762055Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:26:11.79519883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:26:16.79562292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:26:21.321338969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:26:21.336526056Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:26:21.795210851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:26:26.795793103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:31.223839424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:26:31.7958542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:26:33.392684994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:26:33.400085251Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T15:26:35.350480406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:26:36.777117537Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:26:36.777603616Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:26:36.795119765Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:26:36.795119977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:26:36.795144353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:26:36.795152369Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:26:36.795171267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:26:36.795310111Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-07T15:26:36.795318911Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:26:36.795336371Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.069µs success=true
level=debug ts=2026-05-07T15:26:36.796489645Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-07T15:26:36.796498651Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
level=debug ts=2026-05-07T15:26:40.302846162Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=warn ts=2026-05-07T15:26:41.795623732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:43.82332521Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-07T15:26:43.823350202Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
ts=2026-05-07T15:26:45.846845601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57672"
level=warn ts=2026-05-07T15:26:46.795361576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:47.315142007Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-07T15:26:50.822289876Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=info ts=2026-05-07T15:26:51.713717688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:26:51.724152745Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:26:51.795700671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:26:54.10096045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:26:54.104938721Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:26:54.328691748Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=warn ts=2026-05-07T15:26:56.794903341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:57.834932335Z caller=checkpoint.go:372 msg="writing series" size="441 kB"
level=debug ts=2026-05-07T15:26:57.834955918Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-07T15:26:59.24825328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:27:01.298843315Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:27:01.341957933Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:27:01.79575269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:04.848288564Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-07T15:27:05.352779405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:27:06.795602539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:27:06.795624679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:06.795659209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:27:06.795670593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:27:08.354033127Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-07T15:27:11.795268467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:11.861538623Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-07T15:27:12.776905351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="143 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:27:12.785804614Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:27:14.584888353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:27:14.590736923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:27:15.367697583Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:27:16.795175166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:18.874332876Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
ts=2026-05-07T15:27:19.015779155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:60406"
level=warn ts=2026-05-07T15:27:21.79561953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:22.380675028Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:27:25.887519612Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-07T15:27:26.795392813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:29.393481068Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:27:31.795664338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:32.016819329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:27:32.899512021Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-05-07T15:27:35.355461079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:27:36.406066292Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-07T15:27:36.777783018Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:27:36.779102064Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:27:36.795592279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:27:36.79561696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:27:36.79562485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:27:36.795652539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:39.912622719Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-07T15:27:41.572218977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:27:41.580545496Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:27:41.795337727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:43.420056417Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=info ts=2026-05-07T15:27:45.219601909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:27:45.229824569Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:27:45.849758191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58080"
level=warn ts=2026-05-07T15:27:46.794827948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:46.927232838Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-07T15:27:46.927256957Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=14
ts=2026-05-07T15:27:49.018530506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:41144"
level=debug ts=2026-05-07T15:27:50.432781399Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T15:27:51.7956581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:53.938944799Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T15:27:56.795224637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:57.445469142Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-07T15:27:59.074330292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:28:00.951990601Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-07T15:28:01.795894798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:04.458274839Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
ts=2026-05-07T15:28:05.357910983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:28:06.792278301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:28:06.795391098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:28:06.795403745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:28:06.795411098Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:28:06.795491299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:07.965243327Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=info ts=2026-05-07T15:28:11.015950921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="143 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:28:11.027282199Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:28:11.471022017Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=warn ts=2026-05-07T15:28:11.795086186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:28:13.248547499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:13.255586153Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:13.255619032Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:28:13.255670502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:13.260290586Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:28:14.977791276Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-07T15:28:14.977814879Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=8
level=warn ts=2026-05-07T15:28:16.794883508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:18.484942814Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
ts=2026-05-07T15:28:19.021099545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35338"
level=warn ts=2026-05-07T15:28:21.795473215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:21.991364762Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:28:25.497561843Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T15:28:25.852870297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:25.861247679Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:28:26.79541824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:29.004180483Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T15:28:31.795023125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:28:32.15876115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:28:32.163292742Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:28:32.510517499Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-07T15:28:34.813285345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50838"
ts=2026-05-07T15:28:35.360367072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:28:36.019480924Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-07T15:28:36.777313194Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:28:36.777336077Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:28:36.794761506Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:28:36.794789008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:28:36.794796897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:28:36.794799745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:37.150534054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:28:37.150582544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:28:37.154485184Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:28:38.366584605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:28:38.373416009Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:28:39.523949053Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=warn ts=2026-05-07T15:28:41.795134521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:43.030286415Z caller=checkpoint.go:372 msg="writing series" size="642 kB"
level=debug ts=2026-05-07T15:28:43.03030933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=debug ts=2026-05-07T15:28:46.535930214Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=warn ts=2026-05-07T15:28:46.795290122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:50.042304448Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:28:51.099171798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:28:51.107616604Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:28:51.795706106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:53.549412868Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=info ts=2026-05-07T15:28:53.791594672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="143 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:28:53.798878002Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:28:54.976198233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:28:54.977890253Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:28:56.795606214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:57.055942552Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=debug ts=2026-05-07T15:29:00.562075898Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=warn ts=2026-05-07T15:29:01.795907207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:04.069335054Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-07T15:29:04.069359716Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-07T15:29:04.81940052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59864"
level=info ts=2026-05-07T15:29:04.887788687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:29:04.894216242Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T15:29:05.362451883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:29:06.795567493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:29:06.795608756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:29:06.795617263Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:29:06.795649501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:07.289376596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:29:07.574916855Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:29:11.082284877Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-07T15:29:11.795699015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:14.58860176Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:29:16.795353041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:29:18.066053357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:29:18.073208782Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:29:18.095270068Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:29:21.601424577Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-05-07T15:29:21.616456336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:29:21.626159768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:29:21.795225438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:25.107466941Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=warn ts=2026-05-07T15:29:26.795179584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:28.614443352Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-07T15:29:29.756124801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:29:29.761798412Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:29:31.795156122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:32.121379288Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-07T15:29:32.12140275Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-05-07T15:29:35.364462477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:29:35.627391754Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:29:36.777015666Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:29:36.778083862Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:29:36.795548573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:29:36.795575518Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:29:36.79558308Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:29:36.795599421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:39.134217014Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-07T15:29:41.504080432Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:29:41.795381213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:42.640221613Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-07T15:29:46.052103272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:29:46.146336896Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-07T15:29:46.269911806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:29:46.274719636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:29:46.79545285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:29:48.718716353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:29:48.725685634Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:29:49.653370434Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-07T15:29:51.795057341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:53.159626023Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:29:56.666595327Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:29:56.795261962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:00.172251913Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-07T15:30:01.794994129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:03.679619975Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
ts=2026-05-07T15:30:04.82287098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:48708"
level=info ts=2026-05-07T15:30:05.122932324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:05.130664105Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:05.130689539Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:30:05.130714325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:05.134002618Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:30:05.366528654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:30:06.46464867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:30:06.471367706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:30:06.471394883Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:30:06.471421565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:30:06.477451045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:30:06.795574003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:30:06.795605555Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:30:06.795613468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:30:06.795654996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:07.185866462Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-07T15:30:07.185887084Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=10
level=debug ts=2026-05-07T15:30:10.692367154Z caller=checkpoint.go:372 msg="writing series" size="922 kB"
level=debug ts=2026-05-07T15:30:10.919933444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:30:11.795703847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:14.198420387Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-07T15:30:15.734488597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:15.738814523Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:30:15.857790378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60092"
level=warn ts=2026-05-07T15:30:16.79544615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:30:17.132684695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:30:17.136653316Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:30:17.70479754Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:30:19.952973139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:30:19.959550683Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:30:21.211348357Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:30:21.795388861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:24.718681122Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-07T15:30:24.718754483Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-07T15:30:26.795847601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:28.224824748Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=debug ts=2026-05-07T15:30:31.731354742Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:30:31.795169852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:35.237282196Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:30:35.369180011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:30:36.777397533Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:30:36.778650097Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-07T15:30:36.778839517Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:30:36.778879868Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975741 n=1
level=debug ts=2026-05-07T15:30:36.77892166Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975741 n=1
level=info ts=2026-05-07T15:30:36.795081871Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:30:36.795101908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:36.795114504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:30:36.795123843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:30:38.63013227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:30:38.755478849Z caller=checkpoint.go:372 msg="writing series" size="8.8 MB"
level=debug ts=2026-05-07T15:30:38.75550991Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.9 MB" series=4
level=debug ts=2026-05-07T15:30:40.747220848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:30:41.795118581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:42.251198172Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T15:30:43.396890069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:30:43.403554769Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:30:43.6520401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:30:43.657017599Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:30:44.333693681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:30:44.340881308Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:30:44.576780236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:44.580758212Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:30:45.757405809Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-07T15:30:46.795806632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:49.264077479Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=warn ts=2026-05-07T15:30:51.795661438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:52.771464715Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:30:52.771486839Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=debug ts=2026-05-07T15:30:56.276350025Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-07T15:30:56.795334713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:59.783534864Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-07T15:31:00.165543656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:31:00.172396264Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:31:01.795665111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:03.289104336Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:31:05.370993612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:31:06.795407214Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:31:06.795435065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:31:06.795443607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:31:06.795456732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:06.795528333Z caller=checkpoint.go:384 msg="flushing series" totalSize="115 kB" series=3
level=info ts=2026-05-07T15:31:06.798905377Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-07T15:31:06.800257512Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004784546s
level=debug ts=2026-05-07T15:31:11.411764234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:31:11.79537702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:31:15.861710945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41544"
level=warn ts=2026-05-07T15:31:16.794853186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:19.789215505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:31:21.795018062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:31:26.734899363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:31:26.74253942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:31:26.796793892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:31:30.179985493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:31:30.184393851Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:31:30.300673093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:31:30.305783167Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:31:31.7950803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:31:35.372520037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:31:36.777143616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:31:36.777683898Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:31:36.79521058Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T15:31:36.795217517Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T15:31:36.795236714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:31:36.795245069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:31:36.795326405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:31:36.795360304Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-07T15:31:36.79536886Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:31:36.795387526Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.755µs success=true
level=debug ts=2026-05-07T15:31:36.797818265Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T15:31:36.797829781Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=debug ts=2026-05-07T15:31:40.302855523Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=warn ts=2026-05-07T15:31:41.795266197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:31:42.418647194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:31:42.428964507Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:31:43.835746155Z caller=checkpoint.go:372 msg="writing series" size="23 MB"
level=debug ts=2026-05-07T15:31:43.835772735Z caller=checkpoint.go:384 msg="flushing series" totalSize="24 MB" series=2
level=debug ts=2026-05-07T15:31:45.480666806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:31:45.859686578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:31:45.863789296Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:31:46.794870283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:47.315130653Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-07T15:31:50.821985079Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-07T15:31:51.794924331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:54.328708541Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
level=warn ts=2026-05-07T15:31:56.794889353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:57.774710619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:31:57.835097313Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=debug ts=2026-05-07T15:32:01.341248285Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:32:01.794860528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:04.848291675Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-07T15:32:04.848311303Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-07T15:32:05.374002483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:32:06.795084244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:32:06.795113029Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:32:06.795120969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:32:06.795177384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:08.354263401Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-07T15:32:11.795354578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:11.861654598Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-07T15:32:13.253680606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.1 MB" avg_comp="2.0 MB" total_uncomp="52 MB" avg_uncomp="17 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:32:13.267293406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:32:15.231494778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:32:15.235812702Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:32:15.367497325Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:32:16.79529226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:18.864096775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:32:18.874386971Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-07T15:32:21.795357943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:22.381051478Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T15:32:23.564002989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:32:23.571518853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:32:23.57154682Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:32:23.571576807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:32:23.577976097Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:32:25.88655999Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:32:26.794824262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:29.393180159Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-07T15:32:29.508473113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:32:29.515242996Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:32:30.20341584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:32:30.207476685Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:32:31.795776256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:32.900298847Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-05-07T15:32:34.831288164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35766"
ts=2026-05-07T15:32:35.375771543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:32:36.406557245Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-07T15:32:36.776928653Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:32:36.777981137Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:32:36.795521578Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:32:36.795547513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:32:36.795555037Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:32:36.795594055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:39.91317397Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:32:41.795356117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:32:43.005159075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:32:43.015383256Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:32:43.419813981Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=warn ts=2026-05-07T15:32:46.795451112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:46.926168321Z caller=checkpoint.go:372 msg="writing series" size="891 kB"
level=debug ts=2026-05-07T15:32:46.926190097Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=12
ts=2026-05-07T15:32:49.035182616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49342"
level=debug ts=2026-05-07T15:32:50.432194117Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-07T15:32:51.12678161Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:32:51.795375974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:52.269535587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:32:52.271226485Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:32:53.939407471Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T15:32:56.795351318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:57.445525141Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=debug ts=2026-05-07T15:33:00.951794822Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-07T15:33:01.79544863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:04.458060329Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
ts=2026-05-07T15:33:05.377623394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=warn ts=2026-05-07T15:33:06.794979308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:33:06.79497733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:33:06.795005497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:33:06.79501407Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:33:07.965065295Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T15:33:08.060315727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:33:08.069508822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:33:11.471678626Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T15:33:11.700200851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:33:11.706893343Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:33:11.795164007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:33:11.916533066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:33:11.920967209Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:33:14.977575314Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-07T15:33:15.065070191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:33:15.069106593Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:33:15.869679306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38926"
level=warn ts=2026-05-07T15:33:16.794880952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:18.164543104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:33:18.48487963Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-07T15:33:18.484902133Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
ts=2026-05-07T15:33:19.037946596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49140"
level=warn ts=2026-05-07T15:33:21.795788061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:21.990782972Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:33:25.49692472Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-07T15:33:26.794875796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:33:28.669140463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:33:28.681512692Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:33:29.003777103Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T15:33:31.79538674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:32.510878665Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-07T15:33:35.379758254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:33:36.017025509Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-07T15:33:36.777184309Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:33:36.777436738Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:33:36.794997799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:33:36.79499888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:36.795025937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:33:36.795033756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:33:39.523773657Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-07T15:33:41.795270361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:43.030103734Z caller=checkpoint.go:372 msg="writing series" size="642 kB"
level=debug ts=2026-05-07T15:33:43.030126757Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-07T15:33:43.063509507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:33:43.559316054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:33:45.871578394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48142"
level=debug ts=2026-05-07T15:33:46.536114782Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-05-07T15:33:46.795196924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:33:49.040129587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49514"
level=debug ts=2026-05-07T15:33:50.043339708Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:33:51.795183256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:53.549412873Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=warn ts=2026-05-07T15:33:56.795418241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:57.055569841Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-07T15:34:00.562667619Z caller=checkpoint.go:372 msg="writing series" size="455 kB"
level=info ts=2026-05-07T15:34:01.335600624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:34:01.341771347Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:34:01.705810726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:34:01.712596061Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:34:01.795068342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:34:03.948404102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:34:03.955339106Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:34:04.068946813Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
ts=2026-05-07T15:34:04.835453135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:49412"
ts=2026-05-07T15:34:05.381743323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:34:06.795091199Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:34:06.795119484Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:34:06.79512865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:34:06.795128019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:07.575032394Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:34:10.500836886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:34:10.504624186Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:34:11.081751629Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-07T15:34:11.081774536Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-07T15:34:11.795774779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:14.588667863Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
ts=2026-05-07T15:34:15.873909224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53800"
level=warn ts=2026-05-07T15:34:16.795311223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:18.095235167Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:34:18.916044562Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:34:18.916075422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="445 kB" avg_comp="445 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:34:18.921865417Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:34:19.041927068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57644"
level=debug ts=2026-05-07T15:34:21.601756099Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=warn ts=2026-05-07T15:34:21.795285908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:25.107801042Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-07T15:34:26.312510775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:34:26.320132928Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:34:26.794823648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:28.614910761Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T15:34:31.795042391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:32.120970703Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=debug ts=2026-05-07T15:34:32.120994296Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-07T15:34:34.917717068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:34:35.38347709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:34:35.626828258Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-07T15:34:36.777326594Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:34:36.777335724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:34:36.794822812Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:34:36.794841752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:36.794854829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:34:36.79486682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:34:37.060066266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:34:37.065055574Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:34:39.133946665Z caller=checkpoint.go:372 msg="writing series" size="566 kB"
level=info ts=2026-05-07T15:34:41.67253066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:34:41.676526659Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:34:41.79508738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:42.64002894Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-07T15:34:45.17898849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:34:45.185765027Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:34:46.147052001Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T15:34:46.795663348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:49.653117871Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-07T15:34:51.795139687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:53.159477435Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:34:54.65162657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:34:56.144552644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:34:56.15523159Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:34:56.665773306Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:34:56.795689832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:00.172982601Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:35:01.795077247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:03.678953606Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
ts=2026-05-07T15:35:05.385279856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:35:05.721854198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:35:05.727437531Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:35:05.727463386Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:35:05.727487845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:35:05.730982269Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:35:06.147385422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:35:06.151782124Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:35:06.794941055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:35:06.794979908Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:35:06.794987851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:35:06.79504789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:07.185790276Z caller=checkpoint.go:372 msg="writing series" size="999 kB"
level=debug ts=2026-05-07T15:35:07.185813691Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=10
level=debug ts=2026-05-07T15:35:10.692903433Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-05-07T15:35:11.795246677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:14.198282972Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-07T15:35:14.1983059Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T15:35:16.795428735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:35:16.850572188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:35:16.857755268Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:35:16.857778942Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:35:16.857804092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:35:16.861990477Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:35:17.705110758Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:35:20.292580534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:35:20.296738854Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:35:21.211614553Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:35:21.795494444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:24.718463672Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-07T15:35:26.151990388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:35:26.795660895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:35:27.317367991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:35:27.323286149Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:35:28.224983483Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-07T15:35:30.283608756Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:35:30.536449894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:35:30.540812937Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:35:31.731272111Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:35:31.794904779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:35.237149429Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:35:35.387829522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:35:36.777119973Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:35:36.777591432Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:35:36.795131583Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:35:36.795163008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:35:36.79517223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:35:36.795186452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:38.756619314Z caller=checkpoint.go:372 msg="writing series" size="9.1 MB"
level=debug ts=2026-05-07T15:35:38.756664683Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.4 MB" series=7
level=info ts=2026-05-07T15:35:41.0899311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:35:41.101718539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:35:41.795818793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:42.250873233Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T15:35:43.442982106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:35:43.450204867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:35:45.524464494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:35:45.528879257Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:35:45.757095075Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-07T15:35:46.795665586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:49.264315041Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T15:35:49.264337809Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T15:35:51.795229672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:52.771201855Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:35:52.771222855Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T15:35:53.879596978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:35:53.88477365Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:35:54.808434795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:35:56.276246688Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:35:56.7953232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:59.782807021Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-07T15:36:01.795318019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:03.289716734Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-07T15:36:03.50452815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:36:03.510532871Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T15:36:05.389570842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:36:06.795363958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:36:06.795381937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:06.79539715Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:36:06.795406223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:36:06.796473064Z caller=checkpoint.go:384 msg="flushing series" totalSize="119 kB" series=3
level=info ts=2026-05-07T15:36:06.798798146Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-07T15:36:06.800946499Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005429669s
level=info ts=2026-05-07T15:36:11.561558207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:11.567518538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:11.567542591Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:36:11.567566289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:11.571063485Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:36:11.794917939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:36:16.795840837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:19.551151049Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:36:19.67462396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:36:21.795384151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:22.768691554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:22.774747887Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:36:26.209939593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:36:26.214275079Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:36:26.795151437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:36:31.795792703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:33.269241833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:36:33.276805692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:36:34.380479099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:36:34.389422961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:36:34.389450023Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:36:34.389476996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:36:34.396023477Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:36:35.391721958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:36:36.776888464Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:36:36.777931194Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:36:36.795486116Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:36:36.795483783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:36:36.795515812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:36:36.795525767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:36:36.795550101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:36.79567567Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-07T15:36:36.795687386Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:36:36.795712556Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.901µs success=true
level=debug ts=2026-05-07T15:36:36.797379362Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T15:36:36.797392445Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-07T15:36:39.196030371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:36:39.201073432Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:36:40.303600919Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=warn ts=2026-05-07T15:36:41.795698781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:43.830956652Z caller=checkpoint.go:372 msg="writing series" size="25 MB"
level=debug ts=2026-05-07T15:36:43.830988191Z caller=checkpoint.go:384 msg="flushing series" totalSize="26 MB" series=2
ts=2026-05-07T15:36:45.880379033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45450"
level=warn ts=2026-05-07T15:36:46.795091683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:47.316331787Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-07T15:36:48.01500648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:36:48.02390308Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:36:49.050923945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:48278"
level=debug ts=2026-05-07T15:36:50.822354026Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-07T15:36:51.795857586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:54.329103038Z caller=checkpoint.go:372 msg="writing series" size="448 kB"
level=debug ts=2026-05-07T15:36:54.600006296Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:36:56.795822969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:57.769589733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:36:57.775543935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:36:57.836018471Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-05-07T15:36:57.836044646Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-07T15:37:00.07762343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:37:00.081878318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:37:01.342245575Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T15:37:01.79488596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:04.848785682Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-07T15:37:05.393835004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:37:06.795768228Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:37:06.795797243Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:37:06.795806999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:37:06.795795685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:08.355001114Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-07T15:37:11.274123425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:37:11.275710339Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:37:11.795335817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:11.861604908Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-07T15:37:15.367914174Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
ts=2026-05-07T15:37:15.882313208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49242"
level=warn ts=2026-05-07T15:37:16.794986694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:37:16.803186638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:37:16.807572261Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:37:18.874561218Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-07T15:37:21.6617001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:37:21.67000312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:37:21.795676247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:22.380708648Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:37:25.886950506Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-07T15:37:26.795021136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:37:27.272979345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:37:27.28077842Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:37:27.818274972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:37:27.822398532Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:37:29.393651133Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-07T15:37:29.593693153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:37:31.795200542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:32.900258593Z caller=checkpoint.go:372 msg="writing series" size="641 B"
ts=2026-05-07T15:37:35.396401575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:37:36.406855719Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-07T15:37:36.777201596Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:37:36.777228928Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=warn ts=2026-05-07T15:37:36.795796271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:37:36.795799806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:37:36.795827138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:37:36.795837291Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:37:38.088818315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:37:38.094006468Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:37:39.912990237Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:37:41.794857112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:43.419661405Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-07T15:37:43.419687894Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-05-07T15:37:45.884866508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55742"
level=warn ts=2026-05-07T15:37:46.795188988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:46.92702978Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T15:37:46.927050825Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-07T15:37:49.676987662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:37:49.683703676Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:37:50.432703579Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T15:37:51.795173456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:53.939752733Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-07T15:37:55.58028968Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:37:56.795380112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:57.445598994Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=debug ts=2026-05-07T15:38:00.95227402Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-07T15:38:01.642325853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:38:01.650401141Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:38:01.795300619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:04.459214117Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-07T15:38:05.39899213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:38:06.795439435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:38:06.795466597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:38:06.795474385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:38:06.795472937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:06.983258704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:38:07.965442554Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=debug ts=2026-05-07T15:38:07.965468055Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-07T15:38:11.47186357Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-07T15:38:11.795840006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:38:14.188482763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:38:14.197023006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:38:14.978512697Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:38:15.887270829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51898"
level=warn ts=2026-05-07T15:38:16.794919695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:38:17.228523134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:38:17.235009991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:38:18.485045497Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=info ts=2026-05-07T15:38:21.557030748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:38:21.562059179Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:38:21.795085408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:21.991109159Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:38:25.497441093Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T15:38:26.795210547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:29.00488573Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-07T15:38:29.459140522Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:38:29.522344303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:38:29.528646255Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:38:31.795610386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:32.511330193Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-07T15:38:35.401484415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:38:36.016927718Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-07T15:38:36.777133717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:38:36.777569342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:38:36.795034995Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:38:36.795060265Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:38:36.795067834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:38:36.795084634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:39.524398163Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=debug ts=2026-05-07T15:38:39.524425992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-07T15:38:41.795041918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:38:42.747569337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:38:42.754547432Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:38:43.030934296Z caller=checkpoint.go:372 msg="writing series" size="642 kB"
level=debug ts=2026-05-07T15:38:46.536691114Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-07T15:38:46.794888978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:38:48.650997385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:38:48.655230312Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:38:50.043708431Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:38:51.795384727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:53.549850544Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-07T15:38:54.803924786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:38:56.795470538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:57.05650641Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-07T15:38:57.056534595Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-07T15:38:57.74000613Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:39:00.562768985Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=warn ts=2026-05-07T15:39:01.795845449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:39:01.862949491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:39:01.871905642Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:39:04.069390422Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-07T15:39:04.251774686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:39:04.255819965Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:39:05.403679603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:39:06.795424575Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:39:06.7954536Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:39:06.795461793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:39:06.795481403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:07.575524809Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:39:09.876858514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:39:09.885331723Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:39:09.885357778Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:39:09.885390059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:39:09.892916197Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:39:11.08234932Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=warn ts=2026-05-07T15:39:11.795269586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:14.588736021Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:39:16.795490947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:18.095390164Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T15:39:20.368659846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:39:20.382845612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:39:21.602107252Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-07T15:39:21.602128591Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-07T15:39:21.795083125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:22.773758279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:39:25.10799235Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-07T15:39:26.795746047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:39:27.833969273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:39:27.84059184Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:39:28.573833503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:39:28.581817556Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:39:28.614985633Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-07T15:39:31.795406143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:32.121753076Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=info ts=2026-05-07T15:39:35.39602828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:39:35.400268482Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:39:35.405966436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:39:35.630170387Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:39:35.630191349Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=debug ts=2026-05-07T15:39:36.776711872Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:39:36.777757507Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:39:36.795273823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:39:36.795307748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:39:36.795319054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:39:36.795422323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:39.134054226Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=warn ts=2026-05-07T15:39:41.795361403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:42.640572488Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-07T15:39:46.147068903Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T15:39:46.79549134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:39:48.276306253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:39:48.281747611Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:39:48.803704065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:39:48.811258352Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:39:49.059721304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55198"
level=debug ts=2026-05-07T15:39:49.539614649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:39:49.65405315Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-07T15:39:51.795260532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:53.160423437Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:39:56.666424669Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:39:56.795214283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:57.233812184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:40:00.173683429Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-07T15:40:01.795243302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:03.679434588Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-07T15:40:04.759600051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:40:04.768581647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:40:05.407848088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:40:06.795762137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:40:06.795789091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:40:06.795796881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:40:06.795818515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:07.186719982Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-07T15:40:07.186741026Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=9
level=info ts=2026-05-07T15:40:08.814828382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:40:08.819841974Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:40:10.692673203Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=warn ts=2026-05-07T15:40:11.795823266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:14.199395201Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T15:40:16.795847341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:17.705214847Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:40:21.211977803Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:40:21.795261014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:24.71886102Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-07T15:40:24.71888276Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-07T15:40:26.795513384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:40:27.417903167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:40:27.422762052Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:40:28.225148723Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-07T15:40:29.511857527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:40:29.515792928Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T15:40:29.515816285Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:40:29.515839103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:40:29.519030535Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:40:30.448009398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:40:31.691655032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:40:31.698622386Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:40:31.731661352Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:40:31.795274978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:40:33.433840038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:40:33.43824369Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:40:35.238262919Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:40:35.410437628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:40:36.777126836Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:40:36.777139633Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:40:36.795649152Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:40:36.795682065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:40:36.795691698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:40:36.795695646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:38.039219911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:40:38.754582406Z caller=checkpoint.go:372 msg="writing series" size="9.6 MB"
level=debug ts=2026-05-07T15:40:38.754606739Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=4
level=warn ts=2026-05-07T15:40:41.795837799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:42.251588756Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T15:40:43.740345952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:40:43.745282678Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:40:45.757727508Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-07T15:40:46.359462746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:40:46.366283974Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:40:46.794831002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:49.264374367Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T15:40:49.264400346Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T15:40:51.795535416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:52.771484605Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:40:52.771507285Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T15:40:55.631925995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:40:55.637281705Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:40:56.277166939Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:40:56.795183317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:57.731944392Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:40:59.783555549Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-07T15:41:01.79533451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:03.289703824Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-07T15:41:04.428290963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:41:04.432598599Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:41:04.857870269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55452"
ts=2026-05-07T15:41:05.412391476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:41:06.795682456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:41:06.795688787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:06.79570982Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:41:06.795717774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:41:06.796747893Z caller=checkpoint.go:384 msg="flushing series" totalSize="124 kB" series=3
level=info ts=2026-05-07T15:41:06.800700386Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-07T15:41:06.802215304Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006326068s
level=warn ts=2026-05-07T15:41:11.795849864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:41:16.795444639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:41:17.779787098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:41:17.786588529Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:41:19.06587104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58234"
level=info ts=2026-05-07T15:41:20.122990433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:41:20.129890837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:41:21.795089833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:41:26.795544162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:27.729387571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:41:31.408609611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:31.415732199Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:31.415756938Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:41:31.415783212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:31.419493069Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:41:31.795333585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:41:34.85971725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39472"
ts=2026-05-07T15:41:35.414418584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:41:35.6943022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:41:35.701986702Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:41:36.777123134Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:41:36.77713436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:41:36.795600089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:41:36.795624878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:41:36.795646668Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:41:36.795627042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:41:36.795616555Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:41:36.79583246Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-07T15:41:36.795845696Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:41:36.795872094Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.372µs success=true
level=debug ts=2026-05-07T15:41:36.798281719Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T15:41:36.798298081Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=debug ts=2026-05-07T15:41:39.870823786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:41:39.872783423Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:41:40.303612194Z caller=checkpoint.go:372 msg="writing series" size="715 kB"
level=warn ts=2026-05-07T15:41:41.795271604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:41:42.203830853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:42.208516922Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:42.20854527Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:41:42.208569024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:42.212535112Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:41:43.826366219Z caller=checkpoint.go:372 msg="writing series" size="26 MB"
level=debug ts=2026-05-07T15:41:43.826390368Z caller=checkpoint.go:384 msg="flushing series" totalSize="27 MB" series=2
ts=2026-05-07T15:41:45.89842574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60802"
level=warn ts=2026-05-07T15:41:46.794934226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:47.315985402Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-07T15:41:50.763492225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:50.76930634Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:41:50.822768545Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-07T15:41:51.212893975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:41:51.21696801Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:41:51.795099143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:53.096950946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:41:53.097025934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="943 kB" avg_comp="943 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:41:53.101595834Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:41:54.328831508Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=warn ts=2026-05-07T15:41:56.795754661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:57.83614671Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=debug ts=2026-05-07T15:41:57.836170902Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=4
level=info ts=2026-05-07T15:42:00.98366822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:42:00.989152609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:42:01.341981509Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T15:42:01.795674323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:04.848206338Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
ts=2026-05-07T15:42:05.416521877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:42:05.694994173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:42:05.704503054Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:42:06.79505105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:42:06.795079004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:42:06.795086682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:42:06.795105598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:08.355526203Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-07T15:42:11.685290161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:42:11.691199676Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:42:11.79577322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:11.862038889Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-07T15:42:13.610424298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:42:13.614007297Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:42:15.368130074Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
ts=2026-05-07T15:42:15.90026394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33456"
level=warn ts=2026-05-07T15:42:16.795688862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:17.744297813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:42:18.874463156Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-07T15:42:21.348749594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:42:21.35380248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:42:21.673852064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:42:21.681098904Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:42:21.795646991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:22.380799371Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:42:23.599929568Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:42:25.887576408Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:42:26.795444483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:29.394150304Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:42:31.795825006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:32.900198257Z caller=checkpoint.go:372 msg="writing series" size="641 B"
ts=2026-05-07T15:42:35.418991511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:42:36.407740001Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-07T15:42:36.407761972Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-07T15:42:36.777399461Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:42:36.777941615Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:42:36.795327142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:42:36.795332293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:36.795354093Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:42:36.795361796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:42:39.913415257Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-07T15:42:40.569266185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:42:40.575891033Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:42:41.795600668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:43.41991074Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-07T15:42:45.591938816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:42:45.600566901Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:42:46.79504628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:46.926910399Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T15:42:46.926931742Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-07T15:42:48.186337398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:42:48.190468778Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:42:48.83317311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:42:48.839762799Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:42:48.839789937Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:42:48.839817041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:42:48.845581834Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:42:49.069818043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40782"
level=debug ts=2026-05-07T15:42:50.432883616Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T15:42:51.795702545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:53.939330982Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-07T15:42:54.046794272Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:42:56.795277599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:57.445951063Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-07T15:43:00.952483586Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-07T15:43:01.728589274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:43:01.74210636Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:43:01.795674631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:43:03.896969848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:43:03.904129864Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:43:04.459158077Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
ts=2026-05-07T15:43:05.421467787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:43:06.795431498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:43:06.795460211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:43:06.795444919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:43:06.795468147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:43:07.966179744Z caller=checkpoint.go:372 msg="writing series" size="315 kB"
level=info ts=2026-05-07T15:43:08.529566412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:43:08.53394428Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:43:08.846649344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:08.850698162Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:08.850723425Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:43:08.850746263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:08.854026088Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:43:11.472256327Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-07T15:43:11.795383539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:14.978983719Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:43:15.90425914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38064"
level=warn ts=2026-05-07T15:43:16.795801721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:17.329829337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:43:18.485269154Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-07T15:43:18.48530013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-05-07T15:43:19.07268898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33352"
level=info ts=2026-05-07T15:43:20.938888664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:43:20.945051017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:43:21.795702103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:21.991580508Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-07T15:43:22.873351196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:43:22.880198378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:43:24.63584273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:24.639107959Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:43:25.498189073Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-07T15:43:26.795299207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:29.004324938Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-07T15:43:29.330096503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:43:29.99026736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:43:29.994435505Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:43:31.795352032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:32.510686725Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-07T15:43:34.86513711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43488"
ts=2026-05-07T15:43:35.423765831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:43:36.01749013Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-07T15:43:36.77725654Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:43:36.777247849Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:43:36.7955991Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:43:36.795628735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:43:36.795648804Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:43:36.795663039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:39.523885711Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-07T15:43:41.795558677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:43.0305643Z caller=checkpoint.go:372 msg="writing series" size="642 kB"
level=debug ts=2026-05-07T15:43:43.030588453Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-07T15:43:43.787842985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:43:43.796032004Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:43:45.444956532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:43:45.452875138Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:43:45.906167395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58792"
level=debug ts=2026-05-07T15:43:46.537320823Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-07T15:43:46.795618873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:43:49.074899447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:47842"
level=debug ts=2026-05-07T15:43:50.043046531Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:43:50.613672714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:43:50.617988567Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:43:51.795670885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:53.549583449Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:43:56.369267806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:43:56.795764763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:57.056909842Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-07T15:44:00.562758433Z caller=checkpoint.go:372 msg="writing series" size="422 kB"
level=warn ts=2026-05-07T15:44:01.795424069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:01.814887809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:44:02.022844933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:44:02.028266871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:44:03.231921812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:44:03.237035909Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:44:04.068985392Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
ts=2026-05-07T15:44:05.426468296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:44:06.79558369Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:44:06.795614414Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:44:06.795623703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:44:06.79564564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:44:07.214131632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:44:07.220739372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:44:07.575692909Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:44:11.082046265Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-07T15:44:11.082067584Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-07T15:44:11.795230107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:14.588782814Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-07T15:44:15.580819104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:44:15.587830181Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:44:15.908344807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53350"
level=warn ts=2026-05-07T15:44:16.795493649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:18.095868882Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:44:21.601713724Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=warn ts=2026-05-07T15:44:21.795598352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:25.10832461Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-07T15:44:25.622030444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:44:26.795359802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:28.615156833Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=warn ts=2026-05-07T15:44:31.795239479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:32.121705981Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=debug ts=2026-05-07T15:44:32.12172891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T15:44:34.328127545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:44:34.332962538Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:44:35.428860164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:44:35.628358061Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:44:35.628379804Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=debug ts=2026-05-07T15:44:36.777074822Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:44:36.777109815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:44:36.795420793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:44:36.795445622Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:44:36.795452786Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:44:36.795481652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:39.134138621Z caller=checkpoint.go:372 msg="writing series" size="493 kB"
level=info ts=2026-05-07T15:44:39.615558101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:44:39.623708808Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:44:41.795014214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:42.046363594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:44:42.641341756Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-07T15:44:42.726556772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:44:42.730902623Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:44:43.591486018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:44:43.595900976Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T15:44:45.910869365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56306"
level=debug ts=2026-05-07T15:44:46.147124342Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T15:44:46.795582035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:49.653906649Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-07T15:44:51.795526916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:53.160463142Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T15:44:54.769751704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:44:54.779260673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:44:56.666678133Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:44:56.795362116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:00.172870464Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-07T15:45:00.677274734Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:45:01.795055553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:01.99233882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:45:01.996581375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:45:03.680014311Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-07T15:45:04.875622925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:45:04.879812904Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:45:05.431133844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:45:06.795125435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:45:06.795157305Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:45:06.795168089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:45:06.795187769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:07.186147585Z caller=checkpoint.go:372 msg="writing series" size="913 kB"
level=debug ts=2026-05-07T15:45:07.186169663Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=9
level=debug ts=2026-05-07T15:45:10.692899591Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-05-07T15:45:11.795720845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:14.199532729Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-07T15:45:14.199556684Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-05-07T15:45:15.913197286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56954"
level=info ts=2026-05-07T15:45:16.555277088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:45:16.561617869Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:45:16.795158206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:17.706153464Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:45:21.211927149Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:45:21.794853622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:24.719297069Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T15:45:26.795740855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:28.224949299Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-07T15:45:28.971269254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:45:28.981806513Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:45:28.98183147Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:45:28.981863446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:45:28.98880411Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:45:30.247808493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:45:30.247867555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:45:30.252949395Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T15:45:30.253962344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:45:30.258011492Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:45:31.731771742Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:45:31.795314008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:35.238272272Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:45:35.433660639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:45:36.77721717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:45:36.777269154Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=debug ts=2026-05-07T15:45:36.779055539Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:45:36.779098259Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975742 n=1
level=debug ts=2026-05-07T15:45:36.779141675Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975742 n=1
level=info ts=2026-05-07T15:45:36.795600629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:45:36.79562546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:45:36.795644206Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:45:36.795721731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:38.756024846Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-07T15:45:38.756045736Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=7
level=info ts=2026-05-07T15:45:39.323872758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:45:39.328580241Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:45:40.344213684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:45:40.34590023Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:45:41.794858141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:42.251086187Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=debug ts=2026-05-07T15:45:45.757860783Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-07T15:45:45.861303147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:45:45.868890428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:45:45.915510067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59296"
level=info ts=2026-05-07T15:45:45.99322801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:45:45.997452692Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:45:46.598295266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:45:46.605807853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:45:46.795717987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:47.394912846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:45:47.399759896Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:45:49.264809822Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T15:45:49.264834339Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T15:45:51.795258854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:52.771657074Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:45:52.771683212Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=debug ts=2026-05-07T15:45:55.773083445Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:45:56.277612683Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:45:56.795653098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:58.051219311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:45:58.058189351Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:45:59.783387716Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-07T15:46:01.795104582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:46:03.217694761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:46:03.225153963Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:46:03.290385575Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:46:04.872557194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40858"
ts=2026-05-07T15:46:05.435663174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:46:06.795362291Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:46:06.795392997Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:46:06.795405422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:46:06.795425368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:06.796436205Z caller=checkpoint.go:384 msg="flushing series" totalSize="130 kB" series=3
level=info ts=2026-05-07T15:46:06.798482215Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-07T15:46:06.799414941Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.003364508s
level=warn ts=2026-05-07T15:46:11.795225742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:46:16.795351303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:46:16.962746112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:46:16.970840562Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T15:46:17.157036866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:46:17.166712763Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:46:18.331840825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:46:18.336912411Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:46:20.07514889Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:46:21.795078775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:22.82525281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:46:24.776040613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:46:24.781433417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:46:26.795681387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:46:27.750223223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:46:27.754279466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T15:46:27.754300792Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:46:27.754322797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:46:27.757753055Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:46:27.997794917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:46:28.005389257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:46:31.795848483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:46:34.874508155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43934"
ts=2026-05-07T15:46:35.437906465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:46:36.777242306Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:46:36.777285171Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:46:36.795643591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T15:46:36.795659364Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T15:46:36.795672588Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:46:36.795681757Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:46:36.795720021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:46:36.795806103Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-07T15:46:36.795814017Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:46:36.795830329Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.113µs success=true
level=debug ts=2026-05-07T15:46:36.79998402Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T15:46:36.800003345Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=info ts=2026-05-07T15:46:38.951576971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:46:38.958785748Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:46:40.302987077Z caller=checkpoint.go:372 msg="writing series" size="821 kB"
level=info ts=2026-05-07T15:46:41.181540531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:46:41.185793188Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:46:41.7959619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:43.826678396Z caller=checkpoint.go:372 msg="writing series" size="27 MB"
level=debug ts=2026-05-07T15:46:43.826702918Z caller=checkpoint.go:384 msg="flushing series" totalSize="28 MB" series=2
level=warn ts=2026-05-07T15:46:46.794847335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:47.316076168Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-07T15:46:49.085280411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54818"
level=info ts=2026-05-07T15:46:49.184159328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:46:49.188628644Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:46:50.822658316Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:46:51.79569123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:54.329056504Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
level=debug ts=2026-05-07T15:46:54.874885991Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:46:56.795717319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:46:57.606907329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:46:57.614347252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:46:57.835323026Z caller=checkpoint.go:372 msg="writing series" size="595 kB"
level=debug ts=2026-05-07T15:46:57.835346095Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-07T15:47:01.342279557Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T15:47:01.794880944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:03.113232851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:47:03.118742129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:47:04.436541382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:47:04.441804312Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:47:04.848265227Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
ts=2026-05-07T15:47:05.440146852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:47:06.795721385Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:47:06.795756102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:47:06.795767061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:47:06.795785185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:08.199282512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:47:08.204586451Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:47:08.354997671Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-07T15:47:11.79576989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:11.862060085Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-07T15:47:12.051898411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:47:12.060176392Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:47:15.355278466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:47:15.359758117Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:47:15.367849238Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-07T15:47:16.794973319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:18.875051453Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=debug ts=2026-05-07T15:47:20.638990922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:47:21.795367506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:22.380767251Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-07T15:47:25.887436863Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-07T15:47:26.7952904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:28.04805505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:47:28.053390091Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:47:28.094698948Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:47:29.39433344Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:47:31.795074464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:32.900831792Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-07T15:47:34.878031673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:52116"
ts=2026-05-07T15:47:35.442755544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:47:36.407499311Z caller=checkpoint.go:372 msg="writing series" size="293 kB"
level=debug ts=2026-05-07T15:47:36.407520741Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-07T15:47:36.777286485Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:47:36.777302121Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:47:36.795657541Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:47:36.795687966Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:47:36.795696064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:47:36.795701713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:39.732128196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:39.7377811Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:39.737802182Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:47:39.7378249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:39.74105186Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:47:39.913865482Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:47:41.795903397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:43.420487761Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=warn ts=2026-05-07T15:47:46.79546537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:46.927304126Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T15:47:46.927326184Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
ts=2026-05-07T15:47:49.089043576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58454"
level=info ts=2026-05-07T15:47:49.182958378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:47:49.193662277Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:47:50.43286383Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-07T15:47:51.242114396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:47:51.246925106Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:47:51.794820918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:52.029518073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:47:52.034545804Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T15:47:53.31649975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:47:53.323809075Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:47:53.939025861Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-07T15:47:55.77056391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:47:55.77585848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:47:55.775879876Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:47:55.775905688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:47:55.779914712Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:47:56.59208699Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:47:56.794990086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:57.44612442Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-07T15:47:58.949550997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:58.953430566Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:47:59.365286643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:47:59.369125291Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:48:00.952770331Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-07T15:48:01.79570794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:03.764322292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:48:04.458623882Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
ts=2026-05-07T15:48:05.445029521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:48:06.794968744Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:48:06.794994143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:06.795009781Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:48:06.795019773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:48:07.536875581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:07.54307589Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:48:07.966055871Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=debug ts=2026-05-07T15:48:11.472425968Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-07T15:48:11.472450036Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-07T15:48:11.795520724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:14.978716775Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
ts=2026-05-07T15:48:15.924269977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59118"
level=warn ts=2026-05-07T15:48:16.795202423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:18.484606832Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
ts=2026-05-07T15:48:19.091394159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58400"
level=info ts=2026-05-07T15:48:20.175682241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:48:20.183417448Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:48:21.795270966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:21.991309141Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-07T15:48:24.410830748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:48:24.415813166Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:48:25.49836409Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-07T15:48:26.795463563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:29.004607414Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T15:48:31.795443946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:32.165873368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:48:32.510656528Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T15:48:33.401303445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:33.406467832Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:48:34.34895097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:48:34.353865177Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T15:48:34.881159304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:44456"
ts=2026-05-07T15:48:35.447787393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:48:36.017393671Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-07T15:48:36.776888528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:48:36.777933543Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:48:36.795333721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:48:36.795356908Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:48:36.795364938Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:48:36.795386391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:39.523898915Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-07T15:48:41.367485556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:41.372266724Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:41.372289872Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:48:41.372313515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:41.376112001Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:48:41.795481896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:43.030358149Z caller=checkpoint.go:372 msg="writing series" size="642 kB"
level=debug ts=2026-05-07T15:48:43.030381063Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
ts=2026-05-07T15:48:45.926448276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52324"
level=debug ts=2026-05-07T15:48:46.536843685Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=warn ts=2026-05-07T15:48:46.795846892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:48:47.883376883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:48:47.890455205Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:48:50.043022591Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:48:50.879091461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:50.888244372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:50.888269767Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:48:50.888299838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:50.895075632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:48:51.795564986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:48:52.026389208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:52.031274898Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:48:53.54952769Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:48:56.795455039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:57.056523613Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-07T15:48:57.91469169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:48:57.918967297Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:48:58.9209362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:49:00.563392752Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-07T15:49:00.563414442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-07T15:49:01.795758642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:04.069056859Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-07T15:49:04.810311343Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:49:05.449777743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:49:06.518780028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:49:06.522810538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:49:06.795750463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:49:06.795779308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:49:06.79578699Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:49:06.795787186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:49:07.033852931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:49:07.040088932Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:49:07.576012663Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:49:11.083919357Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=warn ts=2026-05-07T15:49:11.795196817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:49:12.113825105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:49:12.118401167Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:49:14.588614353Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:49:16.79522517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:18.095212768Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:49:19.095555333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46960"
level=debug ts=2026-05-07T15:49:21.601791242Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
level=warn ts=2026-05-07T15:49:21.795447868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:25.108553468Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-07T15:49:25.108580895Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-07T15:49:25.484757491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:49:25.493776776Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:49:26.79561922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:28.614480462Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-07T15:49:30.957181671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:49:30.962311518Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T15:49:31.11385506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:49:31.118487855Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:49:31.795776293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:32.121472992Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-05-07T15:49:32.911209649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:49:32.92003683Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:49:33.939845514Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:49:35.451592578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:49:35.628757409Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:49:35.628782941Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-07T15:49:36.777265574Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:49:36.777274243Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:49:36.795670023Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:49:36.79569872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:49:36.795710087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:49:36.795720303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:39.134573992Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=debug ts=2026-05-07T15:49:41.476192841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:49:41.477779779Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:49:41.795069008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:42.641107437Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-07T15:49:44.925965868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:49:44.934339121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:49:46.147235923Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T15:49:46.795425305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:49:49.097836953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55100"
level=debug ts=2026-05-07T15:49:49.653926579Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T15:49:51.753764097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:49:51.757800434Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:49:51.795189051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:53.160399019Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T15:49:56.666365621Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:49:56.795138372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:58.640968665Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:50:00.173068553Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:50:01.486755637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:01.49233951Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:01.492362836Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:50:01.49238682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:01.495393211Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:50:01.795617912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:02.833622533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:50:02.837611665Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:50:03.679987272Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
ts=2026-05-07T15:50:05.453660302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:50:06.795612901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:50:06.79565739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:50:06.795650058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:06.795667377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:50:07.186996898Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-07T15:50:07.187019105Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=9
level=debug ts=2026-05-07T15:50:10.693469274Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T15:50:10.69349345Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-07T15:50:11.795116147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:14.199337203Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T15:50:16.795468079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:17.705768495Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:50:18.923879407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:50:18.932568344Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T15:50:19.09976726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57972"
level=info ts=2026-05-07T15:50:19.282852701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:19.286614639Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:50:21.211681371Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:50:21.795627103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:22.475914241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:50:22.480989653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:50:24.718849219Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-07T15:50:24.933597466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:50:24.941331981Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:50:26.301206168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:50:26.795360489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:27.219283194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:50:28.225350472Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-07T15:50:28.501214918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:50:28.505524129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:50:31.731797197Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:50:31.795649976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:50:34.888757345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50304"
level=debug ts=2026-05-07T15:50:35.23836046Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:50:35.4554987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:50:36.777170995Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:50:36.777447426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:50:36.794760172Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:50:36.794789663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:50:36.794798522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:50:36.794804669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:38.755538514Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-07T15:50:38.755561525Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=8
level=info ts=2026-05-07T15:50:39.999318291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:50:40.00960354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:50:41.795352334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:42.250925925Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T15:50:44.464335247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:50:44.468504085Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:50:45.757462104Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-07T15:50:46.795206471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:49.264395219Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T15:50:49.264418714Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T15:50:51.795162246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:52.388400366Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:50:52.771201684Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:50:52.7712249Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T15:50:54.850968704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:54.857771186Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:50:55.288541894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:50:55.292905367Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:50:56.276971373Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:50:56.794902721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:59.783680035Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T15:51:01.795410914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:03.290563274Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:51:04.890866692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40602"
ts=2026-05-07T15:51:05.45757425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:51:06.795744785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:51:06.795774264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:51:06.795782537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:51:06.795800802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:06.79681014Z caller=checkpoint.go:384 msg="flushing series" totalSize="133 kB" series=3
level=info ts=2026-05-07T15:51:06.800600734Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-07T15:51:06.801440652Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005491512s
level=info ts=2026-05-07T15:51:06.891284887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:51:06.899735812Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:51:10.451659126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:51:11.795526831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:51:13.236015828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:51:13.243070336Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:51:16.795684617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:17.640093269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:51:19.103888309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39466"
level=info ts=2026-05-07T15:51:20.180317348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:51:20.185113232Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:51:21.795810242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:51:24.695810823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:51:24.700649536Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:51:25.058709161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:51:25.06336996Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:51:26.795788771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:51:31.795816383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:51:34.89326717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45916"
ts=2026-05-07T15:51:35.45941942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:51:36.777506311Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:51:36.778009185Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:51:36.795352135Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T15:51:36.795365466Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T15:51:36.795376117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:51:36.795387246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:51:36.795420312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:51:36.795507226Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-07T15:51:36.795515388Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:51:36.795531317Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.211µs success=true
level=debug ts=2026-05-07T15:51:36.797459808Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T15:51:36.797470742Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-07T15:51:37.791684433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:51:37.798259546Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:51:39.818102531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:51:39.824897825Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:51:40.303024134Z caller=checkpoint.go:372 msg="writing series" size="697 kB"
level=warn ts=2026-05-07T15:51:41.795075481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:43.832817176Z caller=checkpoint.go:372 msg="writing series" size="28 MB"
level=debug ts=2026-05-07T15:51:43.832846618Z caller=checkpoint.go:384 msg="flushing series" totalSize="29 MB" series=2
level=warn ts=2026-05-07T15:51:46.795786171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:47.316298837Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-07T15:51:47.740620046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:51:47.746179359Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:51:47.746202039Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:51:47.746226008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:51:47.750593548Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:51:47.868381744Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:51:49.447630494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:51:49.45177688Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:51:50.822756295Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-07T15:51:51.795795984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:54.328927233Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=warn ts=2026-05-07T15:51:56.794819715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:57.83523082Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-05-07T15:51:57.835256457Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-07T15:51:59.399320765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:51:59.405937414Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T15:52:00.388528943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:52:00.395572561Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:52:00.395597692Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:52:00.395627319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:52:00.402149355Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:52:01.341575227Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T15:52:01.795323534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:52:04.79698806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:52:04.801614872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:52:04.848820172Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
ts=2026-05-07T15:52:05.461695751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:52:05.978518758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:52:06.79496591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:52:06.794993329Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:52:06.794984056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:52:06.79500165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:52:08.354368523Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-07T15:52:11.795589571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:11.860827779Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-07T15:52:15.36768207Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-07T15:52:15.90800468Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:52:16.795228925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:52:17.694814837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:52:17.708247855Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:52:18.874355845Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
ts=2026-05-07T15:52:19.108177659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55694"
level=info ts=2026-05-07T15:52:20.674365371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:52:20.679310003Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:52:21.582094654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:52:21.589203408Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:52:21.79513559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:22.380788835Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T15:52:22.496092317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:52:22.500489103Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:52:25.887165609Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-07T15:52:26.647024854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:52:26.651991301Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T15:52:26.79483273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:29.393950724Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-07T15:52:30.380785777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:52:30.385207422Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:52:31.795051737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:32.900375302Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T15:52:34.39562125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:52:34.400090243Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:52:35.463575016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T15:52:36.407037865Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-07T15:52:36.407061443Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-07T15:52:36.777321541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:52:36.77733039Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:52:36.795681951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:52:36.795705679Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:52:36.795714032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:52:36.795731723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:52:38.303865177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:52:38.313201755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:52:39.912898606Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-07T15:52:41.277777099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:52:41.795508025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:43.420014738Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=warn ts=2026-05-07T15:52:46.795181862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:46.927206647Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T15:52:46.927229637Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-07T15:52:50.432536393Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T15:52:51.795555028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:52:53.159881823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:52:53.167808218Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:52:53.93868944Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-07T15:52:56.795264235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:57.445311247Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=debug ts=2026-05-07T15:52:59.763426488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:53:00.951752625Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-07T15:53:01.795107154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:04.458345Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
ts=2026-05-07T15:53:05.465269976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:53:06.795214148Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:53:06.795242082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:53:06.795252329Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:53:06.795259952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:07.964863254Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=info ts=2026-05-07T15:53:08.298761976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:53:08.305284965Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T15:53:08.305307738Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:53:08.305332328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:53:08.308701979Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:53:11.47186581Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-07T15:53:11.471889035Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-07T15:53:11.795133005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:13.810836122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:53:14.977924006Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-07T15:53:15.941452363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55060"
level=warn ts=2026-05-07T15:53:16.794872525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:18.484696967Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
ts=2026-05-07T15:53:19.111154005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35762"
level=info ts=2026-05-07T15:53:19.708109593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:53:19.714617418Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:53:19.80208749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:53:19.806097247Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:53:20.560050547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:53:20.566929324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:53:21.102112555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:53:21.105782286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:53:21.44380115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:53:21.449508857Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T15:53:21.794855821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:53:21.90815811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:53:21.912677221Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:53:21.991235005Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:53:25.49802869Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-05-07T15:53:26.794846755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:53:27.078005536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:53:27.086688526Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:53:29.004295067Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T15:53:31.795066408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:32.511088085Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
ts=2026-05-07T15:53:35.467963017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:53:36.01697701Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-07T15:53:36.777142004Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:53:36.777504044Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:53:36.794856677Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:53:36.794857911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:36.794884209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:53:36.794892078Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:53:39.523424664Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-07T15:53:41.795690953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:43.030117084Z caller=checkpoint.go:372 msg="writing series" size="546 kB"
level=debug ts=2026-05-07T15:53:43.030139177Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-05-07T15:53:44.342305669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:53:44.344056936Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:53:45.943670675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48002"
level=debug ts=2026-05-07T15:53:46.536959425Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=warn ts=2026-05-07T15:53:46.794880755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:48.354312186Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:53:49.80578152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:53:49.810665702Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:53:50.043137574Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:53:51.795036099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:53.549279552Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-07T15:53:56.795163578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:57.056274073Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-07T15:53:57.385820762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:53:57.390346523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:54:00.562445835Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=debug ts=2026-05-07T15:54:00.562472933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-07T15:54:01.795163246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:04.069251512Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-07T15:54:04.58667015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:54:04.590611653Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:54:04.986834435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:04.990752335Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:04.990773613Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:54:04.990795782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:04.993828947Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:54:05.470403642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:54:06.795226007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:54:06.795255069Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:54:06.795263611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:54:06.7952559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:07.575834371Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:54:11.081587908Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=warn ts=2026-05-07T15:54:11.795693535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:54:12.140859875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:54:12.148151992Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:54:12.901584428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:54:12.909529669Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:54:14.588790169Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-07T15:54:15.952395695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:54:15.957063423Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:54:15.993416334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:54:15.997973115Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T15:54:16.795710037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:54:17.593707981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:17.597883454Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:54:18.095327034Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:54:21.601306599Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=warn ts=2026-05-07T15:54:21.794902845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:22.297832534Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:54:25.108166866Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=debug ts=2026-05-07T15:54:25.108190177Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-05-07T15:54:26.795100563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:28.614613523Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-07T15:54:31.795072615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:32.121445398Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
ts=2026-05-07T15:54:35.472171808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:54:35.628045326Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:54:35.62806683Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=debug ts=2026-05-07T15:54:36.087371065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:54:36.777112091Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:54:36.777444944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:54:36.794745458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:54:36.79477899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:54:36.794787073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:54:36.79486978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:39.134275667Z caller=checkpoint.go:372 msg="writing series" size="603 kB"
level=warn ts=2026-05-07T15:54:41.795182705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:42.640132943Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-07T15:54:43.709533099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:54:43.716683048Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:54:43.716711654Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:54:43.716734922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:54:43.720674136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:54:46.146512181Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-07T15:54:46.272917164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:54:46.276465567Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:54:46.795124717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:49.321949336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:54:49.653916152Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T15:54:51.031214679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:54:51.035224574Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:54:51.795430687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:53.160296687Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:54:54.990742469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:54.994579207Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:54:55.065672144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:54:55.072049649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:54:56.66681489Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:54:56.795621686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:00.172906735Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-07T15:55:00.964825631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:00.969972393Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:55:01.795393291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:03.490312176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:03.494952225Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:55:03.679548546Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-07T15:55:04.577355155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:55:04.582956968Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T15:55:04.904119547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41948"
ts=2026-05-07T15:55:05.474761828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:55:06.795096102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:55:06.795135006Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:55:06.795142595Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:55:06.795126893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:07.186419026Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T15:55:07.18644115Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=9
level=debug ts=2026-05-07T15:55:10.693255419Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:55:10.693279582Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-07T15:55:11.350769303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:55:11.357976003Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:55:11.795744821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:14.199094949Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T15:55:16.795719342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:17.705707407Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-07T15:55:19.116275995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57386"
level=debug ts=2026-05-07T15:55:19.450240266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:55:21.212352379Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:55:21.795246676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:24.718383104Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-07T15:55:25.159701471Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:55:26.795739565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:28.225315108Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-07T15:55:31.731710524Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=warn ts=2026-05-07T15:55:31.795115202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:55:34.906417992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:37294"
level=debug ts=2026-05-07T15:55:35.238007293Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-07T15:55:35.476503301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T15:55:36.028004359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:55:36.03341365Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:55:36.776905265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:55:36.777969772Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:55:36.795360255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:55:36.795386632Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:55:36.795394284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:55:36.795413352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:38.755196716Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-07T15:55:38.755222276Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=8
level=warn ts=2026-05-07T15:55:41.795881277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:42.250791377Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T15:55:42.494476227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:42.49981917Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:55:45.451024232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:55:45.454782869Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:55:45.757744349Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-07T15:55:46.795244131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:47.023210017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:55:47.028296973Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:55:49.1182748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:32820"
level=debug ts=2026-05-07T15:55:49.264750552Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T15:55:49.264773496Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T15:55:51.794909344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:52.083224875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:52.090263568Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:52.090287024Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:55:52.090312544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:52.096457988Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:55:52.510668132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:55:52.518866859Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:55:52.771438846Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:55:52.771460803Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T15:55:55.968544889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:55.973532655Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:55.973554308Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:55:55.9735784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:55.97751198Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:55:56.277253377Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:55:56.795709961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:56.831724744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:55:57.800421424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:55:59.728757133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:55:59.733683119Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:55:59.783152015Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-07T15:56:01.795359716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:03.289716023Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-07T15:56:05.478509244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:56:06.744685273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:56:06.75042008Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:56:06.794972197Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:56:06.794977699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:06.795001463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:56:06.795009537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:56:06.7961392Z caller=checkpoint.go:384 msg="flushing series" totalSize="138 kB" series=3
level=info ts=2026-05-07T15:56:06.800486121Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-07T15:56:06.801434643Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005759263s
level=info ts=2026-05-07T15:56:10.177681024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:56:10.185117059Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:56:11.795229529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:56:15.950774597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47452"
level=warn ts=2026-05-07T15:56:16.794887471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:21.778686749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:21.785565419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:21.78558645Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:56:21.785607953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:21.78886459Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:56:21.794936812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:56:26.795062472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:27.769380553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:56:27.862774165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:56:27.86704387Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:56:30.080800328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:56:30.085610284Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:56:31.795117525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:32.097681243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:56:32.103388525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:56:33.62244603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:56:33.626439247Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:56:34.747544545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:34.751186738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:56:34.909458384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56726"
ts=2026-05-07T15:56:35.480400518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T15:56:36.776798851Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:56:36.777857235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:56:36.79479271Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T15:56:36.794824781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:36.794841704Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:56:36.794858123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:56:36.794865956Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:56:36.794923853Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-07T15:56:36.794933508Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:56:36.794953869Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.754µs success=true
level=debug ts=2026-05-07T15:56:36.797495307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:56:36.798120354Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T15:56:36.798130688Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=debug ts=2026-05-07T15:56:40.302943078Z caller=checkpoint.go:372 msg="writing series" size="796 kB"
level=warn ts=2026-05-07T15:56:41.79556359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:43.825134668Z caller=checkpoint.go:372 msg="writing series" size="28 MB"
level=debug ts=2026-05-07T15:56:43.825160535Z caller=checkpoint.go:384 msg="flushing series" totalSize="29 MB" series=2
level=info ts=2026-05-07T15:56:44.14699906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:56:44.154174634Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:56:45.51640398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:56:45.522886567Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T15:56:45.953476151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52796"
level=warn ts=2026-05-07T15:56:46.794898776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:47.31534407Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-07T15:56:50.821630431Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-07T15:56:51.795589207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:53.056130922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:56:54.328341958Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=warn ts=2026-05-07T15:56:56.795659924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:57.269556331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:56:57.27789573Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:56:57.835158596Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-05-07T15:56:57.835181946Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=debug ts=2026-05-07T15:57:01.341275119Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T15:57:01.795058123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:04.847776808Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-05-07T15:57:05.482609655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T15:57:06.795662778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:57:06.795697121Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:57:06.795707131Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:57:06.795731467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:08.354106411Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-07T15:57:11.795058188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:11.860206257Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-07T15:57:13.013346944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:57:13.019182482Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:57:15.367030714Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
ts=2026-05-07T15:57:15.955836175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50074"
level=info ts=2026-05-07T15:57:15.982682501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:57:15.986887231Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:57:16.795079105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:18.873510197Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
ts=2026-05-07T15:57:19.123229292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42192"
level=warn ts=2026-05-07T15:57:21.795702358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:22.380255177Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T15:57:23.373270176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:57:23.37720863Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:57:23.529040018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:57:23.530729827Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:57:24.182135487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:57:24.186570963Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:57:25.691259124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:57:25.695779442Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:57:25.886530755Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-07T15:57:26.795443577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:28.06257617Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:57:29.393352918Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:57:31.795376741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:32.899966719Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-07T15:57:35.484776298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:57:36.406053471Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-07T15:57:36.406074961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-07T15:57:36.777541993Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:57:36.778692858Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:57:36.794994773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:57:36.795021783Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:57:36.795029381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:57:36.795123481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:37.088560735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:57:37.097979336Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:57:39.912499857Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-07T15:57:41.795367019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:42.100537727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:57:42.10753804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:57:43.419447601Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=warn ts=2026-05-07T15:57:46.794819166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:46.925959813Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-07T15:57:46.925984074Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-07T15:57:49.297560429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:57:49.302906495Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:57:50.43234746Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T15:57:51.795488078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:53.938628818Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-07T15:57:56.698930882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:57:56.702873047Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:57:56.795250571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:57.445165192Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=debug ts=2026-05-07T15:57:57.52440023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:57:57.524392819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="46 kB" avg_uncomp="46 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:57:57.528294551Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T15:57:57.825334533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:57:57.828418377Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:57:58.125290461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:57:58.128287315Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T15:57:58.425153744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:57:58.428271055Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T15:57:58.725243817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:57:58.728297092Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T15:57:59.025295567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="32 kB" avg_uncomp="32 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:57:59.028501994Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T15:57:59.325244785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T15:57:59.328354448Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T15:57:59.625320543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:57:59.628445138Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T15:57:59.924682227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:57:59.92779877Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T15:58:00.951432101Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-07T15:58:01.794970187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:02.202023721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:58:04.458101317Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
ts=2026-05-07T15:58:05.486520108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=warn ts=2026-05-07T15:58:06.795488497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:06.795482897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:58:06.795514432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:58:06.795525322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:58:07.964604512Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=debug ts=2026-05-07T15:58:07.964625692Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T15:58:09.213611307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:58:09.219689522Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T15:58:10.251308379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:58:10.254346083Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T15:58:10.388818142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:58:10.391600094Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T15:58:11.19699217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:58:11.200561684Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T15:58:11.47145474Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T15:58:11.481477651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:58:11.48505618Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:58:11.734683068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T15:58:11.737624886Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T15:58:11.795177917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:13.084108475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:58:13.087678162Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:58:13.307204008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:58:13.310853857Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:58:14.548687543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:58:14.551888115Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T15:58:14.749465523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:58:14.75236137Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T15:58:14.977412381Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T15:58:16.795114591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:17.609405477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:17.618223026Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:17.618245916Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:58:17.61827675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:17.626857104Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:58:17.990992964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:58:17.994068702Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T15:58:18.484605333Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=info ts=2026-05-07T15:58:18.933862873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:58:18.93838998Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T15:58:19.127083643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:43320"
level=info ts=2026-05-07T15:58:21.12798777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:58:21.134026898Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:58:21.795235062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:21.990212888Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-07T15:58:24.159107768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:58:25.497476531Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-05-07T15:58:26.795488408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:29.003574162Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-07T15:58:31.309102833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:58:31.312674481Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T15:58:31.795098564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:32.510086239Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T15:58:33.193291122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:33.201496746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:58:35.488646273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:58:35.503114282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T15:58:35.506220625Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T15:58:36.016333769Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=debug ts=2026-05-07T15:58:36.777025925Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T15:58:36.778073692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T15:58:36.795403177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:58:36.795406817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:36.795432768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:58:36.795442691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:58:39.26141559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:58:39.266270197Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T15:58:39.523053788Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-05-07T15:58:41.207460842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:58:41.214354205Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:58:41.7952599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:42.832990492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:58:42.836144096Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T15:58:43.029959733Z caller=checkpoint.go:372 msg="writing series" size="658 kB"
level=debug ts=2026-05-07T15:58:43.029981964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=10
level=info ts=2026-05-07T15:58:45.055592991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:58:45.060096725Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:58:45.060119552Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:58:45.060143284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:58:45.064071756Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T15:58:45.960969726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33274"
level=info ts=2026-05-07T15:58:46.182352474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:58:46.185193686Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T15:58:46.535733059Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=warn ts=2026-05-07T15:58:46.794816261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:46.805977134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:58:48.784096647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:58:48.787301257Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T15:58:48.852538227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:58:48.855535806Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-07T15:58:49.128749515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42692"
level=debug ts=2026-05-07T15:58:50.042940851Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:58:51.490787439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:58:51.495297765Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:58:51.795292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:52.040145898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:58:52.043092485Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:58:52.195020354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:58:52.198014115Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T15:58:53.549380203Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-07T15:58:56.795374333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:57.055676836Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=debug ts=2026-05-07T15:58:59.870975094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:59:00.561821308Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=debug ts=2026-05-07T15:59:00.561844888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-07T15:59:01.795148141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:03.684515055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:59:03.691595887Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T15:59:04.068994004Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
ts=2026-05-07T15:59:05.490759004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T15:59:06.795246099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:59:06.795271798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:59:06.795279714Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:59:06.795299231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:07.442684811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:59:07.446595536Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:59:07.575175133Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:59:08.49085791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:59:08.497775767Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:59:09.819186921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:59:09.825868521Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:59:10.120657075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:59:10.124143553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:59:11.081991723Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=warn ts=2026-05-07T15:59:11.79503078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:14.587612355Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-07T15:59:16.794806132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:17.664357808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:59:17.667968788Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T15:59:17.826561077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:59:17.830905367Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:59:18.094721365Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:59:19.131240616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58158"
level=debug ts=2026-05-07T15:59:21.6008578Z caller=checkpoint.go:372 msg="writing series" size="458 kB"
level=debug ts=2026-05-07T15:59:21.600880464Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-07T15:59:21.795240079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:24.013206953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T15:59:24.017575784Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T15:59:25.107191827Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-07T15:59:25.534525791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:59:25.538733733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T15:59:26.541533779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T15:59:26.546229958Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T15:59:26.795595125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:28.61374423Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=debug ts=2026-05-07T15:59:29.910506024Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:59:29.963562494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:59:29.967145579Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T15:59:31.795389299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:32.120795571Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=info ts=2026-05-07T15:59:33.592479472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:59:33.596307988Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:59:34.918435732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55044"
ts=2026-05-07T15:59:35.493310944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T15:59:35.627469578Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:59:35.627491722Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-07T15:59:36.777237117Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:59:36.777767627Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=warn ts=2026-05-07T15:59:36.795168446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:36.79516556Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:59:36.795193421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:59:36.79520436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:59:37.023320296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:59:37.026685145Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T15:59:37.861514816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:59:37.868505434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:59:39.133366759Z caller=checkpoint.go:372 msg="writing series" size="633 kB"
level=info ts=2026-05-07T15:59:40.037865117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:59:40.042417326Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T15:59:41.518986748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:59:41.52222205Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T15:59:41.795195255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:42.639697834Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-07T15:59:42.731930045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:59:42.735917458Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T15:59:46.145962947Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-07T15:59:46.148216421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:59:46.577988128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:59:46.581111486Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T15:59:46.795017419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:59:49.133427484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38956"
level=debug ts=2026-05-07T15:59:49.652553997Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T15:59:50.92652773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:59:50.930428617Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:59:51.795172991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:52.293612699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:59:52.298494271Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:59:53.158817787Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-07T15:59:56.665659234Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:59:56.795511623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:00.171831108Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=warn ts=2026-05-07T16:00:01.794928355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:03.324178939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:00:03.32424854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="120 kB" avg_uncomp="120 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:00:03.332052031Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:00:03.679224017Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-07T16:00:04.519859634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:00:04.527849995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T16:00:04.920161884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55274"
ts=2026-05-07T16:00:05.495260098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:00:06.795460014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:00:06.795492636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:00:06.795501058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:00:06.795526362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:07.18566285Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:00:07.185685023Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=9
level=info ts=2026-05-07T16:00:07.342371503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:00:07.346614184Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02f1c83e%3A19e03124736%3A614fee3e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T16:00:07.3466528Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:00:07.346676835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:00:07.349196566Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-07T16:00:07.908965065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:00:07.912839697Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:00:10.691944146Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:00:10.69196985Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-07T16:00:11.365765413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:00:11.371220112Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:00:11.795514822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:14.19859601Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-07T16:00:16.795167007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:17.704262355Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:00:18.170854231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:00:18.175130333Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:00:18.397836792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:00:18.401295005Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-07T16:00:21.069906087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:00:21.073099812Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:00:21.16019693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:00:21.167823537Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:00:21.21106385Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T16:00:21.795206512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:22.452204483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:00:22.455675627Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:00:24.041751784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:00:24.044978184Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:00:24.718010084Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-07T16:00:25.615171469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:00:25.618311369Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:00:26.066681325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:00:26.071193942Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:00:26.795697122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:28.224193302Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T16:00:30.844664198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:00:30.849697273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:00:30.853565932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:00:31.73098359Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:00:31.795773756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:31.828490651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:00:33.554577601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:00:33.559291002Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:00:33.559320783Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:00:33.559352616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:00:33.563319269Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:00:34.263285132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:00:34.267044186Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:00:35.237417545Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-07T16:00:35.497604648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:00:35.534768993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:00:35.541070853Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:00:36.385832797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:00:36.38922257Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:00:36.776803786Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:00:36.777867168Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:00:36.779226977Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T16:00:36.779275972Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975743 n=1
level=debug ts=2026-05-07T16:00:36.77932439Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975743 n=1
level=info ts=2026-05-07T16:00:36.795278191Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:00:36.795320887Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:00:36.795331467Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:00:36.7953544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:37.21092447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:00:37.214474299Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:00:38.756176423Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-07T16:00:38.756210823Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=warn ts=2026-05-07T16:00:41.795014829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:42.250056076Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T16:00:43.247856088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:00:43.253182885Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:00:45.09288987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:00:45.10200103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:00:45.756901456Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-07T16:00:46.795504318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:48.232157001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:00:48.237199507Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:00:48.665475637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:00:48.669696947Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:00:49.263456213Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:00:49.263480401Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-07T16:00:51.141523438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:00:51.145095839Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:00:51.795544956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:52.7590669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:00:52.764095316Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:00:52.770547633Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:00:52.770571643Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=debug ts=2026-05-07T16:00:56.276391373Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:00:56.795864899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:56.841030952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:00:56.844593918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-07T16:00:59.783003788Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-07T16:01:01.79532301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:03.289471793Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:01:04.348175192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:01:04.352072481Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-07T16:01:04.923751178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55256"
ts=2026-05-07T16:01:05.499851939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:01:05.583083684Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:01:06.794996499Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:01:06.795025107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:01:06.79503693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:01:06.79502825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:06.796120059Z caller=checkpoint.go:384 msg="flushing series" totalSize="104 kB" series=3
level=info ts=2026-05-07T16:01:06.952854819Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-07T16:01:06.953783556Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.158701491s
level=info ts=2026-05-07T16:01:11.651569949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:01:11.655957841Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:01:11.795922677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:12.620878503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:01:12.622984482Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:01:13.681915808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:01:13.686668293Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:01:14.785943901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:01:14.792852693Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:01:15.267537149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:01:15.271309587Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:01:15.722140574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:01:15.729853579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:01:16.794823722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:17.677200601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:17.683268242Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:17.68329484Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:01:17.683323083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:17.689390531Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:01:18.370461259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:01:18.374184029Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:01:21.795029217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:22.233297104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:01:22.237572643Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:01:26.100466347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:01:26.104374821Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:01:26.795240652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:29.354538474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="28 MB" avg_comp="2.8 MB" total_uncomp="232 MB" avg_uncomp="23 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:01:29.362433417Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:01:30.045299525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:01:30.049652848Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:01:30.753354551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:30.758524945Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:01:31.795147176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:34.26436098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:01:34.267893473Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:01:35.502155231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:01:36.387683913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:01:36.392772714Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:01:36.777183489Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:01:36.777542563Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:01:36.794836557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:01:36.794866817Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:01:36.794857627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:36.794874877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:01:36.794881039Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:01:36.794970701Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-07T16:01:36.794977325Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:01:36.794993465Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.447µs success=true
level=debug ts=2026-05-07T16:01:36.796571041Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T16:01:36.796587145Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=debug ts=2026-05-07T16:01:36.911617436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:01:40.213811282Z caller=checkpoint.go:372 msg="writing series" size="952 kB"
level=info ts=2026-05-07T16:01:41.448354293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:01:41.45421949Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:01:41.795739566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:41.815898111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:01:41.81916958Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:01:42.47114574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:01:42.474577111Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:01:43.66232062Z caller=checkpoint.go:372 msg="writing series" size="31 MB"
level=debug ts=2026-05-07T16:01:43.662347525Z caller=checkpoint.go:384 msg="flushing series" totalSize="32 MB" series=2
ts=2026-05-07T16:01:45.970247475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42770"
level=warn ts=2026-05-07T16:01:46.794854129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:47.049114316Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-07T16:01:50.467108978Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-07T16:01:51.79532748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:53.884929988Z caller=checkpoint.go:372 msg="writing series" size="460 kB"
level=info ts=2026-05-07T16:01:54.46017002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:01:54.468101003Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:01:56.202075188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:01:56.20583658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-07T16:01:56.795273572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:56.804382931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:56.809876818Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:01:57.302174237Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=debug ts=2026-05-07T16:01:57.302202267Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-07T16:01:58.766996823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:01:58.772021429Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:02:00.719882826Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-07T16:02:01.361685288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:02:01.365229054Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:02:01.450529103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:02:01.455523548Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:02:01.795863793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:04.137423437Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-07T16:02:04.928441551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38890"
level=info ts=2026-05-07T16:02:05.427898054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:02:05.435200932Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T16:02:05.504242166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:02:06.422787586Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:02:06.795364455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:02:06.795402551Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:02:06.795413889Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:02:06.795434531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:07.197683644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:02:07.202031654Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:02:07.555075174Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-07T16:02:10.972564093Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T16:02:11.794920507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:12.287238107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:02:12.291184036Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:02:12.934606574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:02:13.13513712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:02:13.139393144Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:02:14.072760283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:14.076506883Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:02:14.390592596Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-07T16:02:15.094762111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:02:15.098205315Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:02:16.063832154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:02:16.071537485Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:02:16.794881656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:17.808172667Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-07T16:02:21.226680688Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:02:21.795668102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:24.052774289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:02:24.056838777Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:02:24.080039427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:02:24.083446297Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:02:24.644018149Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-07T16:02:26.795752084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:26.963010092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:02:26.967955131Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:02:28.061875193Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-07T16:02:31.47926137Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:02:31.79536937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:34.534598188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:02:34.539560264Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:02:34.539587384Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:34.539611518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:02:34.543737889Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:02:34.897106606Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-07T16:02:34.897131553Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=debug ts=2026-05-07T16:02:34.941239959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:02:35.506701968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:02:36.777114941Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:02:36.777124639Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:02:36.795524798Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:02:36.795560576Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:02:36.795580512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:02:36.795583674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:38.314388143Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:02:39.481800778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:02:39.485908983Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:02:41.732402816Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=warn ts=2026-05-07T16:02:41.794852867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:42.509499092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:02:42.512996064Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:02:45.151342056Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-07T16:02:45.151365005Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=warn ts=2026-05-07T16:02:46.795105364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:48.300088185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:02:48.303929007Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:02:48.568276509Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-07T16:02:50.491859962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:02:50.497062153Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:02:51.795402294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:51.949875797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:02:51.953621409Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:02:51.985370774Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T16:02:52.352225185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:02:52.357710396Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:02:53.093989806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:02:53.097227775Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-07T16:02:54.873343715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:02:54.876941795Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:02:54.954033449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:02:54.957941886Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:02:55.287199543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:02:55.290579034Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:02:55.403830656Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-07T16:02:56.237669518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:02:56.242831715Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:02:56.795305993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:58.777352856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:58.782280302Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:58.782301142Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:58.782326339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:58.785215751Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:02:58.821404384Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-07T16:02:59.390007608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:02:59.39712577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:02:59.397149205Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:59.397178165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:02:59.404668981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:03:01.795072703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:02.238793206Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=debug ts=2026-05-07T16:03:02.430291234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:03:04.932241443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60812"
ts=2026-05-07T16:03:05.508687569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:03:05.656961379Z caller=checkpoint.go:372 msg="writing series" size="381 kB"
level=info ts=2026-05-07T16:03:06.794760266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:03:06.79479386Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:03:06.794803622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:03:06.794813363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:07.582576895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:03:07.590240111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:03:08.246792113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:03:08.249986768Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:03:08.938626503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:03:08.941964336Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:03:09.074726908Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-07T16:03:09.074755159Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-07T16:03:09.711259048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:03:09.71899525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:03:10.121528426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:03:11.795066345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:12.491785312Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=info ts=2026-05-07T16:03:14.010896536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:03:14.014270289Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:03:14.191895986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:03:14.195342779Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:03:15.910227096Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
ts=2026-05-07T16:03:15.975333213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36970"
level=warn ts=2026-05-07T16:03:16.795353863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:03:19.145353022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39432"
level=debug ts=2026-05-07T16:03:19.327216039Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-07T16:03:21.79567249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:22.744782315Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-07T16:03:26.163375195Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-07T16:03:26.680990106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:03:26.686663139Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:03:26.795351771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:29.580855033Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T16:03:30.20341871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:03:30.209561254Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:03:31.795449207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:31.846546769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:03:31.852515166Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:03:32.907345497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:03:32.912250576Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:03:32.998611168Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:03:34.134684996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:03:34.145708124Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:03:34.920498938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:03:35.511023173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:03:36.416548267Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-05-07T16:03:36.776753523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:03:36.777803016Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:03:36.79516474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:03:36.79519224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:03:36.795202444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:03:36.795195704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:37.822222448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:03:37.827155058Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:03:39.548141632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:03:39.552184407Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:03:39.833718148Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=debug ts=2026-05-07T16:03:39.833740054Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-05-07T16:03:40.18546593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:03:40.189415603Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:03:40.590389762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:03:40.593535202Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:03:41.79537056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:42.334042008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:03:42.33782172Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-07T16:03:43.251445449Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-07T16:03:43.602003382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:03:43.605830898Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-07T16:03:45.977010458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58972"
level=debug ts=2026-05-07T16:03:46.669415417Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-07T16:03:46.795099399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:47.867190483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:03:47.87390258Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:03:50.086646634Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=warn ts=2026-05-07T16:03:51.795205378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:53.465761159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:03:53.471052224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:03:53.504353852Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-07T16:03:54.24405571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:03:54.258157799Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:03:54.499923135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:03:54.503811661Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:03:56.795737229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:56.922168825Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=info ts=2026-05-07T16:03:57.917008573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:03:57.920670823Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:04:00.340024949Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=warn ts=2026-05-07T16:04:01.79526602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:03.758060032Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-07T16:04:03.75808263Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-07T16:04:04.344957586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:04:04.349946294Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:04:04.935857567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:53782"
ts=2026-05-07T16:04:05.513845841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:04:06.227840359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:04:06.595989284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:04:06.599515076Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:04:06.79565973Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:04:06.795690532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:04:06.795679112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:06.795699351Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:04:07.175904401Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:04:09.016417735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:04:10.59308851Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=warn ts=2026-05-07T16:04:11.795693076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:12.510766566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:04:12.515979618Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:04:14.010748672Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-07T16:04:15.979034177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39044"
level=warn ts=2026-05-07T16:04:16.794849315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:17.428611887Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:04:17.864050525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:04:17.867483222Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:04:17.867504429Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:17.867524798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:04:17.870021221Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:04:18.076821277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:04:18.080008461Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:04:18.080029248Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:18.168267897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:04:18.17142002Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-07T16:04:19.148807413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38682"
level=debug ts=2026-05-07T16:04:20.846214965Z caller=checkpoint.go:372 msg="writing series" size="492 kB"
level=warn ts=2026-05-07T16:04:21.795387093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:24.263677635Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-07T16:04:25.3473337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:04:25.350882843Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02f1c83e%3A19e02f1c83f%3A19efc5bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:04:25.350902059Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:25.350925073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="442 kB" avg_uncomp="442 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:04:25.353607431Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:04:26.043129842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:04:26.048302531Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:04:26.787088615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:04:26.790373865Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:04:26.790391984Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:26.790414653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:04:26.793034075Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:04:26.79518308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:27.681732767Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-07T16:04:27.681755941Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-07T16:04:29.097207056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:04:29.10011576Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:04:29.859273149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:04:29.862573129Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:04:30.111688185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:04:30.115367393Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-07T16:04:31.099399668Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=warn ts=2026-05-07T16:04:31.795815393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:32.190444611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:04:32.193622155Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:04:32.711857287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:04:32.714905975Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:04:33.738311459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:04:33.741260114Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:04:33.741279503Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:33.741300568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:04:33.743894562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:04:33.761094108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:04:34.518520664Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:04:34.518543052Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
ts=2026-05-07T16:04:35.516229865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:04:36.619506375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:04:36.624015669Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:04:36.645066181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:04:36.647986715Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:04:36.648002399Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:36.648021539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:04:36.650686497Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:04:36.777122899Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:04:36.777510035Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:04:36.794804785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:04:36.794827713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:04:36.79483601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:04:36.79487209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:37.465008081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:04:37.469117848Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:04:37.88032145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:37.88776167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:37.887788155Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:37.887822912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:37.894488146Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:04:37.934828414Z caller=checkpoint.go:372 msg="writing series" size="504 kB"
level=debug ts=2026-05-07T16:04:41.352184572Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:04:41.795878137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:44.770488653Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-07T16:04:45.321792056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:04:45.330162143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T16:04:45.981657525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35018"
level=info ts=2026-05-07T16:04:46.330940336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:04:46.333908749Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:04:46.354933885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:04:46.357707583Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:04:46.763933917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:04:46.768608349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:04:46.795104383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:48.188661073Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:04:51.605782488Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:04:51.795621474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:53.349251685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:04:53.355021579Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:04:53.355043067Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:53.355064712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:04:53.357766838Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:04:53.464043493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:04:53.467691488Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:04:54.36917083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:04:54.372504062Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:04:54.399500903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:04:54.403624399Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:04:54.403672774Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:54.40370284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:04:54.407124134Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:04:55.02308729Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-07T16:04:55.998293415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:04:56.001657642Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:04:56.001680802Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:56.001701602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:04:56.005147629Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:04:56.039386328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:56.04730444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:04:56.795074105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:58.441461966Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=warn ts=2026-05-07T16:05:01.795694152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:01.858561854Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-07T16:05:03.821740308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:05:03.825281491Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:05:04.465878139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:05:05.277722062Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T16:05:05.277743966Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=9
level=debug ts=2026-05-07T16:05:05.457651961Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:05:05.518565975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:05:06.795387113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:05:06.795361748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:06.795416514Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:05:06.795426921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:05:07.661561685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:05:07.666044457Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:05:08.402293308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:05:08.405520343Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:05:08.695195185Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T16:05:08.69521741Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-05-07T16:05:11.421863585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:05:11.424968561Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:05:11.617561286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:05:11.620662564Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:05:11.795576859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:12.059703193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:05:12.064391715Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:05:12.112716815Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:05:13.658396458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:05:13.662390769Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:05:15.529834935Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:05:16.700585509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:05:16.704181152Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:05:16.795046573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:18.843523239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:05:18.847327261Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:05:18.947653838Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-07T16:05:19.92356575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:05:19.927537455Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-07T16:05:21.795539074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:22.36506282Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T16:05:23.040000741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:05:23.044676918Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:05:23.333464641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:05:23.338673631Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:05:23.338697539Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:05:23.33872154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:05:23.34315636Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:05:25.782902317Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-07T16:05:26.795212474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:27.285218253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:05:27.289278167Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:05:27.74377535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:05:27.750524954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:05:28.691683272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:05:28.694931661Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:05:29.200599171Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:05:31.795408965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:31.911985759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:05:31.915168729Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:05:32.618137262Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-07T16:05:33.737164727Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:05:34.255965129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:05:34.259735562Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:05:34.545539188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="31 MB" avg_comp="2.8 MB" total_uncomp="256 MB" avg_uncomp="23 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:05:34.552566921Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T16:05:35.521302504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:05:35.842206599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:05:35.848250121Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:05:35.895456047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:05:35.898953887Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:05:36.047909137Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-07T16:05:36.047934754Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=8
level=info ts=2026-05-07T16:05:36.454602036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:05:36.459798504Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:05:36.777100241Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:05:36.777132981Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:05:36.79547Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:05:36.795497184Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:05:36.795505399Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:05:36.795557455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:39.45431652Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-07T16:05:41.795585794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:42.871498198Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-07T16:05:43.444064879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:05:43.445736235Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:05:45.985015338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56512"
level=debug ts=2026-05-07T16:05:46.289712317Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:05:46.28973232Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T16:05:46.795140362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:47.027146709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:05:47.032201012Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:05:49.154479525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55678"
level=debug ts=2026-05-07T16:05:49.707672555Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-07T16:05:51.795326537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:52.127339871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:05:52.132156574Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:05:53.125606642Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:05:53.125628557Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=2
level=debug ts=2026-05-07T16:05:56.542078863Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-07T16:05:56.550191782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:05:56.555194282Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:05:56.795477683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:59.960350417Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-07T16:06:01.741816908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:06:01.747046851Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:06:01.795720847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:03.251519219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:06:03.255141411Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:06:03.37769804Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-07T16:06:05.524058047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:06:06.795382955Z caller=checkpoint.go:384 msg="flushing series" totalSize="110 kB" series=3
level=warn ts=2026-05-07T16:06:06.795411675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:06.795425117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:06:06.79544182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:06:06.795452029Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:06:06.796996505Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-07T16:06:06.798080105Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.002948938s
level=info ts=2026-05-07T16:06:07.319675173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:06:07.323149994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-07T16:06:07.449364615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:06:07.452705869Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:06:08.045669155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:06:09.079525514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:06:09.083091503Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:06:11.795393829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:15.026687434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:06:15.032615461Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:06:15.986823479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46608"
level=info ts=2026-05-07T16:06:16.702058651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:06:16.70551697Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:06:16.795026974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:17.442380526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:06:17.44616672Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:06:17.814390936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:06:17.818156606Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:06:19.156539913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44632"
level=info ts=2026-05-07T16:06:19.253425666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:06:19.256411666Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:06:20.309110428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:06:20.316767402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:06:21.795831785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:24.333361392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:06:24.336791883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:06:26.795385262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:27.785324267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:06:27.788604124Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:06:28.170681438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:06:28.173916896Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:06:29.834087015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:06:29.839140321Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:06:30.897987262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:06:30.904703504Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:06:31.795150287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:32.259867546Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:06:33.237120364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:06:35.526768165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:06:36.777117327Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:06:36.77748517Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:06:36.794748674Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:06:36.794765175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:06:36.79478751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:06:36.794795822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:06:36.794806501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:36.79492325Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-07T16:06:36.794931914Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:06:36.794949776Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.038µs success=true
level=debug ts=2026-05-07T16:06:36.796556354Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-07T16:06:36.796566225Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=1
level=debug ts=2026-05-07T16:06:40.21400487Z caller=checkpoint.go:372 msg="writing series" size="922 kB"
level=warn ts=2026-05-07T16:06:41.795275064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:43.66886792Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:06:43.668896398Z caller=checkpoint.go:384 msg="flushing series" totalSize="35 MB" series=2
ts=2026-05-07T16:06:45.989567829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35294"
level=warn ts=2026-05-07T16:06:46.795800222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:47.048726063Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:06:48.520532814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:06:48.531174694Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:06:49.22234796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:06:49.227236663Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:06:49.366532678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:06:49.370313302Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:06:50.46627918Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-07T16:06:51.795211494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:53.885023829Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=info ts=2026-05-07T16:06:54.198566331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:06:54.202128896Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:06:56.794803458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:56.929160229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-07T16:06:56.932310838Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e02f1a064%3A19e02f1a065%3A14d4a097?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-07T16:06:56.932333191Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:06:56.932356378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:06:56.935336048Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:06:57.302797158Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-07T16:06:57.302822913Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-07T16:06:57.345319688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:06:57.348502333Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:06:58.257345056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:06:58.261228823Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:06:58.2612572Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:06:58.261287962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:06:58.264542436Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:07:00.719744871Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-07T16:07:01.795070628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:02.128246147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:07:02.132886308Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:07:04.137872215Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-07T16:07:04.363272074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:07:04.368179517Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:07:04.438520542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:07:04.946785244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50472"
ts=2026-05-07T16:07:05.529327156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:07:06.79495111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:07:06.794986681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:07:06.794997792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:07:06.795051775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:07.555779213Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-07T16:07:07.88092843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:07:07.884401265Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:07:08.642488048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:07:08.646353823Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:07:10.973366104Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-07T16:07:11.207981877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:07:11.215920899Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:07:11.79579371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:12.971196304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:07:12.974380827Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:07:14.390402447Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-07T16:07:14.783683658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:07:14.787071146Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:07:14.787091511Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:07:14.787114722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:07:14.789442924Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:07:16.040251836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:07:16.043978932Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:07:16.49518121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:07:16.498231374Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:07:16.795492095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:16.930786777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:07:16.934510929Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:07:17.808173006Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-07T16:07:21.226341888Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-07T16:07:21.712771591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:07:21.720897922Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:07:21.795400817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:24.643676915Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-07T16:07:26.146041963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:07:26.149990734Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:07:26.532297624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:07:26.537452064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:07:26.79583381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:28.06142358Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-07T16:07:30.592563372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:07:30.595776907Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:07:31.478742112Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:07:31.795769839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:32.188915824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:07:34.104148035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:07:34.107228422Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:07:34.367980051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:07:34.372004109Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:07:34.896662862Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=debug ts=2026-05-07T16:07:34.896692502Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
ts=2026-05-07T16:07:35.531533926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:07:36.542042862Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:07:36.777019783Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:07:36.778071541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:07:36.795400404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:07:36.795431253Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:07:36.795438651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:07:36.795455624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:37.610022424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:07:37.613452Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:07:38.314521952Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:07:40.722683384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:07:40.725882837Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:07:41.196428812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:07:41.201258652Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:07:41.732698924Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T16:07:41.795124947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:41.994210341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:07:41.997976108Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:07:42.910428636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:07:42.916806896Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:07:45.151733325Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:07:45.151764458Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
ts=2026-05-07T16:07:45.992534571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42048"
level=warn ts=2026-05-07T16:07:46.795507082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:48.567867985Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T16:07:51.795334244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:51.985243571Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-07T16:07:55.403019172Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-07T16:07:56.795085241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:56.910413417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:07:56.916290629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:07:57.041170954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:07:57.04510688Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:07:58.343247647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:07:58.34719178Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:07:58.820769966Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-07T16:08:00.367030779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:08:00.370454507Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:08:01.313108417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:08:01.320428248Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:08:01.795241671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:02.238536158Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=debug ts=2026-05-07T16:08:02.536877252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:08:03.574407434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:08:03.578997854Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-07T16:08:05.533677877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:08:05.656935581Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-07T16:08:05.65695763Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T16:08:06.795546415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:08:06.795575767Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:08:06.795583301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:08:06.795604269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:08.450368052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:08:08.454675588Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:08:08.569327353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:08:08.576473383Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:08:08.595531267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:08:08.599091259Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:08:09.074715896Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:08:09.726831196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:08:09.729826197Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:08:11.795694489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:12.492258406Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-07T16:08:15.909730917Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=warn ts=2026-05-07T16:08:16.795425759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:18.709182092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:08:18.713945107Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-07T16:08:19.162845665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56226"
level=debug ts=2026-05-07T16:08:19.326829769Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-07T16:08:21.795002377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:22.660681093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:08:22.664761127Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:08:22.744986907Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:08:22.953687551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:22.959020482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:22.959042465Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:08:22.959066881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:22.963280968Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:08:26.163141106Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T16:08:26.795113576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:27.947089172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:08:29.581014919Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-07T16:08:31.795161313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:32.730834701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:08:32.735457781Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:08:32.998483679Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:08:34.204009045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:08:34.207513679Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-07T16:08:34.951176441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50942"
level=debug ts=2026-05-07T16:08:35.02293475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:08:35.535924689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:08:35.739251913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:08:35.7423951Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:08:36.415742559Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-07T16:08:36.415765448Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-07T16:08:36.776896918Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:08:36.777954986Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:08:36.79536509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:08:36.79538949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:08:36.795396903Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:08:36.795413898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:36.979048574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:08:36.982465509Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:08:37.587157197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:08:37.592071697Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:08:37.768627487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:08:37.771742559Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:08:39.834115754Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=warn ts=2026-05-07T16:08:41.795473488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:42.683490923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:42.68843416Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:08:43.251140665Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:08:45.330713722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:08:45.334578382Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:08:45.996085297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37402"
level=debug ts=2026-05-07T16:08:46.669313071Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-07T16:08:46.795196032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:47.855071729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:08:47.858554246Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:08:48.89684856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:08:48.900546547Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:08:50.087036334Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=warn ts=2026-05-07T16:08:51.795015771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:53.504230297Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-07T16:08:54.079085681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:08:54.083366152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:08:54.563598621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:08:54.566790976Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:08:54.981292629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:08:54.984451275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:08:56.79509305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:56.922522849Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=debug ts=2026-05-07T16:08:56.922543801Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-07T16:08:58.890184281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:08:58.897863102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:09:00.33997603Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=info ts=2026-05-07T16:09:00.581175707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:09:00.584532827Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:09:01.795054169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:03.757952167Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-07T16:09:04.116998977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:09:04.120050518Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:09:05.5380318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:09:06.795051794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:09:06.795088616Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:09:06.795083926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:06.795099663Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:09:07.175047291Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:09:08.369689758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:09:08.378721467Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:09:09.166853485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:09:10.592864115Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=info ts=2026-05-07T16:09:11.507508517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:09:11.511299421Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:09:11.795462087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:14.011178546Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-07T16:09:14.011232066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-07T16:09:16.795470502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:17.428704832Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:09:18.176967113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:09:18.182295497Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:09:20.846552225Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-07T16:09:21.722107649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:09:21.725054735Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:09:21.795566571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:22.532973247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:09:23.458947459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:09:23.462180155Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:09:23.945003739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:09:23.947929522Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:09:24.264048249Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-07T16:09:24.544749945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:09:24.549233358Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:09:24.549257733Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:24.549285264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:09:24.553149639Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:09:25.789118187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:09:25.794021029Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:09:26.096854439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:09:26.100663817Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:09:26.794985349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:27.681842823Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-05-07T16:09:29.443583577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:09:29.447427213Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:09:30.381735427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:09:30.384517329Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:09:31.0999414Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-07T16:09:31.099964666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-07T16:09:31.795158583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:33.36261772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:09:34.518699313Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:09:34.518719978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-07T16:09:35.540263587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:09:36.776893432Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:09:36.778021138Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:09:36.795394294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:09:36.795432302Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:09:36.795443268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:09:36.795464781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:37.059815158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:09:37.063072526Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:09:37.935221632Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-07T16:09:39.483572915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:39.48794965Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:39.487976013Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:39.488002386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:39.491042371Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:09:39.799125172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:09:39.806135401Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:09:39.80615968Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:39.806185951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:09:39.81304096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:09:40.273666354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:09:40.278293936Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:09:41.352593445Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:09:41.795201644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:43.674680955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:09:43.678075817Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:09:44.770290182Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-07T16:09:45.999160038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34598"
level=warn ts=2026-05-07T16:09:46.794818396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:47.233127951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:09:47.236023581Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:09:47.962923152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:09:47.966094327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:09:48.187883326Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:09:51.60533213Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:09:51.795240356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:51.817346581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:09:51.821686995Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:09:53.230898605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:09:53.234804151Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:09:55.023033351Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-07T16:09:55.037139834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:09:55.045368925Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:09:56.79581126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:56.849957517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:56.854052871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:09:58.441774501Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-07T16:10:01.795123702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:01.859282082Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-07T16:10:02.466506983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:10:02.47596102Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:10:04.392922456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:10:04.955561727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47124"
level=debug ts=2026-05-07T16:10:05.278262939Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:10:05.278286919Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=9
ts=2026-05-07T16:10:05.541919747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:10:06.794840387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:10:06.794882504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:10:06.794891293Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:10:06.7949124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:08.695867291Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T16:10:08.69588951Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-05-07T16:10:08.69998202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:10:08.703215271Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:10:08.750290852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:10:08.753308478Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:10:09.33314675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:10:09.338312711Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:10:11.590405272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:10:11.592273317Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T16:10:11.795278419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:12.112687988Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:10:13.274150717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:10:13.279437368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:10:15.529961593Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:10:16.696656515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:10:16.700120863Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:10:16.795797084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:18.947831039Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-07T16:10:20.65788064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:10:20.662995228Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:10:20.853690027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:10:20.856954111Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:10:21.795005523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:22.365540162Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T16:10:23.945219747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:10:23.950038522Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:10:25.302531647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:10:25.306697104Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:10:25.782783603Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-07T16:10:26.795005754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:27.846045078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:10:27.849673272Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:10:28.55256249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:10:28.555784094Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:10:29.183761635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:29.187071385Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:29.187092139Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:29.187113535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:29.190505863Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:10:29.200679594Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:10:31.023691392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:10:31.030897563Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:10:31.795436253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:32.618464011Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-07T16:10:35.544264303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:10:36.051275153Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-07T16:10:36.051302455Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=8
level=info ts=2026-05-07T16:10:36.777164809Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:10:36.777199471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:10:36.795576113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:10:36.795609525Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:10:36.795620016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:10:36.795696076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:39.453739184Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T16:10:39.468816066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:39.474968204Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:39.474994738Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:39.475017378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:39.477460726Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:10:39.552707011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:10:41.795070995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:42.308016288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:42.311604386Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:10:42.871511341Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-07T16:10:43.477209117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:10:43.481211868Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:10:46.289850518Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:10:46.289873848Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T16:10:46.795203348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:47.891765467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:10:47.895312143Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:10:48.625430539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:10:48.628769734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:10:49.706991971Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-07T16:10:51.795820969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:53.125989622Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:10:53.126015473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-07T16:10:56.248364544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:10:56.253698261Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:10:56.542503781Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-07T16:10:56.781748945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:56.784782275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:10:56.794882843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:56.807994139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:10:56.812588089Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:10:57.333689255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:10:57.337468873Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:10:57.337490145Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:57.337512733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:10:57.340001758Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:10:59.96027519Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:11:00.020836155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:11:00.024285197Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02f1a064%3A19e02f1b61f%3A52a1fb75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:11:00.024305555Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:00.024328326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:11:00.027404132Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:11:00.779956641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:00.78687658Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:11:01.794854423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:03.309580092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:11:03.314230074Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:11:03.377446396Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:11:04.510303819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:11:04.51395274Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:11:04.51397544Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:04.513995851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:11:04.516366647Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:11:04.53898396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:11:04.959835661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46576"
ts=2026-05-07T16:11:05.546720738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:11:06.795672709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:11:06.79567282Z caller=checkpoint.go:384 msg="flushing series" totalSize="116 kB" series=3
level=debug ts=2026-05-07T16:11:06.795701405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:11:06.795709202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:11:06.795776785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:06.800710435Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-07T16:11:06.804253642Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00916531s
level=debug ts=2026-05-07T16:11:08.282284961Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:11:10.747027436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:11:10.750853424Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:11:10.750875764Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:10.750897522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:11:10.753449985Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:11:11.701206996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:11:11.704532248Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:11:11.704554762Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:11.704578563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:11:11.707222346Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:11:11.795100092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:13.752495158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:11:13.757286277Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:11:13.828141247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:11:13.831352424Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:11:13.831373833Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:13.831393626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:11:13.834051103Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-07T16:11:16.004707444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51468"
level=warn ts=2026-05-07T16:11:16.79549697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:17.374689857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:11:17.378413115Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:11:17.454474804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:11:17.457556482Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:11:18.818683903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:11:18.822696553Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:11:21.795844881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:22.090945258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:11:22.094142609Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:11:22.310831246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:11:22.313918175Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:11:23.819049383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:11:23.823188924Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:11:25.016459215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:11:25.019735636Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:11:25.855685785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:11:25.859255509Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:11:26.386844056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:11:26.394504463Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:11:26.795207052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:30.008285712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:11:30.01151811Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:11:31.794972717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:32.896595901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:11:32.899795319Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:11:33.813277996Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:11:34.961882727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60746"
level=info ts=2026-05-07T16:11:35.321767435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:11:35.326240994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-07T16:11:35.549299424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:11:36.777175782Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:11:36.777799984Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:11:36.795170786Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:11:36.795189888Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:11:36.795210705Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:11:36.795203104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:36.795220241Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:11:36.795340115Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-07T16:11:36.795348886Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:11:36.795366684Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.634µs success=true
level=debug ts=2026-05-07T16:11:36.797249399Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-07T16:11:36.797263311Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=1
level=info ts=2026-05-07T16:11:37.010421428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:11:37.014065874Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:11:37.014088979Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:37.014113142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:11:37.016759239Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:11:39.093057778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:11:39.097835169Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:11:40.214341917Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:11:40.214363013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-07T16:11:40.911355798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:11:40.915222766Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:11:41.79500892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:43.660322904Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:11:43.660350488Z caller=checkpoint.go:384 msg="flushing series" totalSize="34 MB" series=1
level=info ts=2026-05-07T16:11:46.137993411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:11:46.141555595Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:11:46.795126946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:46.82899798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:11:47.049166727Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:11:50.450762457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:11:50.454481275Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:11:50.466689025Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-07T16:11:51.522291647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:11:51.527327443Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:11:51.794916106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:52.476139218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:11:52.480692173Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:11:53.83539483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:11:53.83844593Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:11:53.885053581Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
level=info ts=2026-05-07T16:11:54.111214523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:11:54.116196136Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:11:54.11621787Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:54.116242476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:11:54.120616599Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:11:54.346217106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:11:54.349191149Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:11:55.371276334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:55.378580265Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:55.37860398Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:55.378659053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:55.385617284Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:11:55.958017163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:11:55.961051574Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:11:56.795611818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:57.30266729Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-07T16:11:57.302688907Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=debug ts=2026-05-07T16:12:00.720492013Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-07T16:12:01.795243572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:02.803238131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:12:02.80641378Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:12:03.590087513Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:12:04.137728905Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-07T16:12:04.381547467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:12:04.386575983Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T16:12:04.964028236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:49876"
ts=2026-05-07T16:12:05.551895278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=warn ts=2026-05-07T16:12:06.795476439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:06.795466546Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:12:06.795505854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:12:06.79551691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:12:07.555745058Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-07T16:12:08.581295244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:12:08.584730091Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:12:09.253327784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:12:09.26167284Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:12:09.559354278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:12:09.563365963Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:12:10.973492838Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T16:12:11.795079272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:14.391391561Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-07T16:12:15.455032331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:12:15.461764582Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:12:16.795739067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:17.808507989Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-07T16:12:17.883687628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:12:17.892307627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:12:21.226340348Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:12:21.795196467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:24.644060594Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=debug ts=2026-05-07T16:12:25.629987693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:12:26.795016579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:27.450008395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:12:27.453081209Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:12:28.062014234Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-07T16:12:31.48014549Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:12:31.795354933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:33.135127022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:12:33.139060808Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:12:33.16925039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:12:33.17406122Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:12:33.360397057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:12:33.363915023Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:12:33.619428259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:12:33.623386007Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:12:34.452664446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:12:34.89709447Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-07T16:12:34.897115384Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-07T16:12:34.955138754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:12:34.958553245Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:12:34.966342256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42050"
ts=2026-05-07T16:12:35.554497133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:12:36.776665855Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:12:36.777720812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:12:36.79512595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:12:36.795151405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:12:36.79515954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:12:36.79517279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:37.604687642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:12:37.610902826Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:12:37.738054386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:12:37.741130958Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:12:38.314960952Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:12:40.355262324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:12:40.358403397Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:12:40.913272836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:12:40.916461278Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:12:41.732883783Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T16:12:41.795289609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:43.621508177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:12:43.626343574Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:12:45.151479229Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-07T16:12:45.151500301Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-05-07T16:12:45.678057441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:12:45.682390884Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:12:46.010143351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48572"
level=info ts=2026-05-07T16:12:46.642165822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:12:46.649335739Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:12:46.795399448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:48.568114981Z caller=checkpoint.go:372 msg="writing series" size="939 B"
ts=2026-05-07T16:12:49.177324166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52326"
level=info ts=2026-05-07T16:12:51.535197647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:12:51.543307972Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:12:51.795400198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:51.989623594Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T16:12:53.665756782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:12:53.668787134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:12:55.404001278Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=warn ts=2026-05-07T16:12:56.795465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:57.987100994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:12:57.994877307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:12:58.821344297Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-07T16:13:01.795562445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:02.239174437Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
ts=2026-05-07T16:13:04.968138503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43852"
ts=2026-05-07T16:13:05.556694688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:13:05.657052173Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-07T16:13:05.657073639Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-07T16:13:05.718199876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:13:06.794851368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:13:06.794876761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:13:06.794885869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:13:06.794884085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:08.109925077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:13:08.113930806Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:13:09.074704055Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:13:09.692398637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:13:09.69657757Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:13:11.030353065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:13:11.033676638Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:13:11.795876505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:12.492750561Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=debug ts=2026-05-07T16:13:15.910454097Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
ts=2026-05-07T16:13:16.012796476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37458"
level=warn ts=2026-05-07T16:13:16.795435415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:17.984843831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:13:17.988052774Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:13:19.328208466Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-07T16:13:19.421673511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:13:19.425010521Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:13:19.702088464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:13:21.79517139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:22.115539427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:13:22.120062452Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:13:22.638333856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:13:22.641735196Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:13:22.746048368Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:13:23.900521078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:13:23.904789083Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:13:26.117100776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:13:26.120213905Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:13:26.163462071Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T16:13:26.795590798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:29.394545834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:13:29.399433086Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:13:29.581154735Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T16:13:30.190099531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:13:30.195094511Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:13:31.795254438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:32.426595059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:13:32.433909612Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:13:32.541154949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:13:32.544906176Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:13:32.544929714Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:13:32.54495364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:13:32.547953862Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:13:32.998610838Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:13:33.134969369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:13:33.139610591Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:13:33.193989125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:13:33.197228536Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:13:34.312016895Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:13:35.558791559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:13:36.416537264Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-07T16:13:36.416563Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-07T16:13:36.777060121Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:13:36.778080447Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:13:36.795417696Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:13:36.795449717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:13:36.795458027Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:13:36.795495911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:39.834215296Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=warn ts=2026-05-07T16:13:41.795615095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:42.668176709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:13:42.671727972Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:13:43.252411581Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-07T16:13:46.669535374Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-07T16:13:46.795324305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:47.800411084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:13:47.804224712Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:13:48.092983236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:13:48.096881177Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-07T16:13:49.180452383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33870"
level=debug ts=2026-05-07T16:13:50.087247311Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:13:51.187399577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:13:51.191588254Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:13:51.19161164Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:13:51.191651817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:13:51.195407993Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:13:51.79578615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:53.50456211Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:13:54.154452079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:13:54.158807682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:13:54.158831807Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:13:54.158854293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:13:54.161449367Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:13:56.426905628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:13:56.434233328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:13:56.795606617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:56.923085129Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-05-07T16:13:56.92310644Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T16:13:57.830374321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:13:57.834142991Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:14:00.340780795Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=warn ts=2026-05-07T16:14:01.795056861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:03.75847197Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-07T16:14:04.97189018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39392"
ts=2026-05-07T16:14:05.560938775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:14:06.795740375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:14:06.795768189Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:14:06.795756228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:06.795776145Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:14:07.176119894Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:14:07.728354381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:07.732988161Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:14:09.14596866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:14:09.150235844Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:14:09.487197234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:14:09.490261734Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:14:10.288962843Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:14:10.593745445Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-07T16:14:10.593768853Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-07T16:14:11.347766709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:14:11.351460729Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:14:11.795847057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:12.074243371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:14:12.078116133Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:14:13.369741943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:14:13.373126032Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:14:14.011034758Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-07T16:14:14.367992776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:14:14.372479601Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:14:16.795441368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:17.072578061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:14:17.077697392Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:14:17.429538226Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:14:18.108453147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:14:18.112389453Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:14:18.736010194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:14:18.740134945Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:14:19.490496646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:14:19.495295531Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:14:20.087742337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:14:20.090663423Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:14:20.847545524Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-07T16:14:21.493048834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:14:21.494555874Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T16:14:21.795816618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:24.264537863Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-07T16:14:24.456058851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:14:24.466201437Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:14:26.795816344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:27.681950464Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-07T16:14:30.786120265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:14:30.789560225Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:14:31.100616667Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-07T16:14:31.100660324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-07T16:14:31.795481439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:31.951881938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:14:31.955517342Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:14:34.51887778Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:14:34.518901053Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-07T16:14:34.973965305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56346"
level=info ts=2026-05-07T16:14:35.369423877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:35.3735231Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:14:35.563566959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:14:36.776990589Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:14:36.778026347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:14:36.795390464Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:14:36.795422124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:14:36.795431239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:14:36.795498789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:36.994537241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:14:37.935342682Z caller=checkpoint.go:372 msg="writing series" size="670 kB"
level=info ts=2026-05-07T16:14:39.678214654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:14:39.681533294Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:14:41.353196126Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:14:41.795147055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:44.722717333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:14:44.726367891Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:14:44.770694917Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-07T16:14:44.841955847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:14:44.848502328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T16:14:46.016503553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36352"
level=warn ts=2026-05-07T16:14:46.795420851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:48.188478532Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
ts=2026-05-07T16:14:49.183661026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:32806"
level=debug ts=2026-05-07T16:14:51.606548884Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:14:51.795226969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:53.385990237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:14:53.394475595Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:14:55.02369559Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T16:14:56.795255364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:58.44167107Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-07T16:15:01.795781167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:01.85895519Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-07T16:15:02.843758523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:15:02.849288432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:15:02.849312741Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:15:02.849338423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:15:02.853494013Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:15:04.494553936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:15:04.49829052Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-07T16:15:04.975752175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40648"
level=debug ts=2026-05-07T16:15:05.277608212Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:15:05.277657253Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=9
ts=2026-05-07T16:15:05.566132431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:15:06.79498717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:15:06.795018706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:15:06.795026877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:15:06.795058166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:07.42499273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:15:07.428160428Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:15:07.879561456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:15:07.883122877Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:15:08.271341968Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:15:08.302385265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:15:08.305483393Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:15:08.695233287Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:15:08.6952585Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-07T16:15:09.254311107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:15:09.258121551Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:15:10.074854898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:15:10.079491029Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:15:11.795736223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:12.112985117Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:15:12.207003979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:15:12.210239189Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:15:12.49295512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:15:12.497500015Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:15:12.639908581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:15:12.642924751Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:15:15.530099742Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:15:16.795406088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:17.09268316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:15:17.096073307Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:15:17.355063969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:15:17.359770731Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:15:18.947598078Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-07T16:15:19.027997366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:15:19.031937067Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:15:19.513351624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:15:20.275941521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:15:20.285462106Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:15:21.795249222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:22.366298082Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T16:15:25.783628111Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-07T16:15:25.796754008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:15:25.80167152Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:15:26.023137371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:15:26.026474888Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:15:26.795065441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:29.201397369Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:15:31.795114436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:32.618383561Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:15:35.184976405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:15:35.193189635Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:15:35.568482922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:15:36.052099861Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-07T16:15:36.052122555Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=8
level=debug ts=2026-05-07T16:15:36.776737306Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:15:36.777776107Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:15:36.779088713Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T16:15:36.779128072Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975744 n=1
level=debug ts=2026-05-07T16:15:36.77916879Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975744 n=1
level=info ts=2026-05-07T16:15:36.795124861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:15:36.795143903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:15:36.795151223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:15:36.795224223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:37.386822412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:15:37.395091064Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:15:38.317745224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:15:39.454078688Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T16:15:41.686919647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:15:41.692524625Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:15:41.795195864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:42.872415506Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=debug ts=2026-05-07T16:15:46.290932496Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:15:46.290957189Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T16:15:46.795149664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:47.700117415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:15:47.704194642Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:15:49.188298076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44330"
level=debug ts=2026-05-07T16:15:49.707567462Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-07T16:15:51.73390124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:15:51.737361232Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:15:51.795286473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:53.125370684Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:15:53.12539811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-07T16:15:56.346472972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:15:56.350785087Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:15:56.543437647Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:15:56.795054335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:58.04367483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:15:58.047758849Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:15:58.259371413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:15:58.262881384Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:15:59.71120325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:15:59.714832832Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:15:59.960673563Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-07T16:16:01.795886041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:03.045170417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:03.050180585Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:03.050204612Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:03.050227846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:03.054109514Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:16:03.378315267Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:16:03.623372176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:16:03.626491462Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:16:05.376562324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:16:05.380108379Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:16:05.509391411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:16:05.512722634Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:16:05.529884644Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:16:05.570778743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:16:06.795674371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:16:06.79566749Z caller=checkpoint.go:384 msg="flushing series" totalSize="118 kB" series=3
level=debug ts=2026-05-07T16:16:06.795698837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:16:06.795707631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:16:06.795781682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:06.802917529Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-07T16:16:06.805207574Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009679866s
level=info ts=2026-05-07T16:16:06.832951946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:16:06.836969057Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:16:07.314985525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:16:07.323672089Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:16:10.847852823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:16:10.854233074Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:16:11.795418767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:13.13697015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:16:13.140459655Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:16:13.711548486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:16:13.715475977Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:16:16.022340336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57258"
level=warn ts=2026-05-07T16:16:16.795076834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:16:18.290222823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:16:18.696409244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:16:18.69970835Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:16:19.191036292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58690"
level=info ts=2026-05-07T16:16:20.480771265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:20.485190537Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:16:21.7954287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T16:16:26.795161573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T16:16:31.795845193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:33.19928941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:16:33.208454879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:16:33.208478893Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:33.208507124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:16:33.214690489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:16:34.124785915Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:16:34.981119145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59810"
ts=2026-05-07T16:16:35.572750963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:16:36.777451528Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:16:36.777933706Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:16:36.795300196Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T16:16:36.795313232Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T16:16:36.795324245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:16:36.795331888Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:16:36.795355644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:36.79547223Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-07T16:16:36.795480659Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:16:36.795496572Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.578µs success=true
level=debug ts=2026-05-07T16:16:36.797181598Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-07T16:16:36.797192943Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=1
level=info ts=2026-05-07T16:16:38.93396436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:16:38.938498139Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:16:40.214304447Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=info ts=2026-05-07T16:16:41.151963275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:16:41.156739609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:16:41.79589576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:43.229346211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:16:43.23292759Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:16:43.563772166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:43.567290724Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:43.567312239Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:43.567335373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:43.570304493Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:16:43.667607357Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:16:43.667651986Z caller=checkpoint.go:384 msg="flushing series" totalSize="35 MB" series=2
ts=2026-05-07T16:16:46.024691634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49596"
level=info ts=2026-05-07T16:16:46.039222917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:16:46.042565003Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:16:46.795152819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:46.873471096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:16:46.881178031Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:16:47.049915061Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-07T16:16:50.467419679Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-07T16:16:51.367772645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:16:51.372494903Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:16:51.795596376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:52.721560291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:52.725888719Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:16:53.213376369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:16:53.216236867Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:16:53.476064951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:16:53.479099941Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:16:53.885178917Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=info ts=2026-05-07T16:16:54.400445833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:54.404343406Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:16:54.526612953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:16:54.529793759Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:16:54.529815409Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:54.529836335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:16:54.532314078Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:16:56.002359182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:16:56.006109291Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:16:56.08531236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:16:56.092959827Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:16:56.795208534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:16:57.302935861Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-07T16:16:57.30296007Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-07T16:16:58.036927894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:16:58.040423393Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:17:00.301349848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:17:00.304579554Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:17:00.304600405Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:00.304622881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:17:00.307231233Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:17:00.719977788Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-07T16:17:00.92780153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:17:01.795817703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:04.138693877Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-07T16:17:04.293284306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:17:04.296663576Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:17:05.574889544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:17:06.794766955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:17:06.794797449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:17:06.794806216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:17:06.794812452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:07.556140305Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-07T16:17:07.923325522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:17:07.926403025Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:17:08.667497457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:17:08.671555301Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:17:10.973784986Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-07T16:17:11.795824531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:13.613992899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:17:13.617778493Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:17:14.391171416Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
ts=2026-05-07T16:17:16.027183056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38846"
level=warn ts=2026-05-07T16:17:16.795839983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:17.012683749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:17:17.015616216Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:17:17.808779534Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-07T16:17:18.141734694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:17:21.226583307Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:17:21.795624993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:23.882708637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:17:23.890237497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:17:24.048621647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:17:24.0538258Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:17:24.644623822Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-07T16:17:25.761060504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:17:25.76413029Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:17:25.764150851Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:25.764171286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:17:25.766767119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:17:26.795330674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:28.062166586Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-07T16:17:29.466695242Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:17:29.738368537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:17:29.741773177Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:17:29.741794462Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:29.741814473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:17:29.744169332Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:17:30.091228919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:17:30.095666148Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:17:31.479776321Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:17:31.795100175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:34.897862635Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=debug ts=2026-05-07T16:17:34.897884528Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-05-07T16:17:34.984555144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38680"
ts=2026-05-07T16:17:35.576766141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:17:35.720911067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:17:35.725564224Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:17:35.725585841Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:35.725607883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:17:35.728164071Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:17:36.776939734Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:17:36.777544261Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:17:36.794962524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:17:36.794986929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:17:36.794976848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:36.794995075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:17:37.60559585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:17:37.608763938Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:17:37.608783367Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:37.608806584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:17:37.611084756Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:17:37.840957568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:17:37.844114227Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:17:37.934377592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:17:37.938216434Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:17:37.938236425Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:37.93825828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:17:37.941183144Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:17:38.315259991Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:17:38.565683847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:17:38.568663264Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:17:38.639845016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:17:38.644562097Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:17:38.674806833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:17:38.677696468Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:17:39.951349557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:17:39.954479307Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:17:41.732665891Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-07T16:17:41.795133544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:45.151722278Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-07T16:17:45.151744569Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=info ts=2026-05-07T16:17:45.33902725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:17:45.347928804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:17:46.7949081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:46.822002261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:17:46.826590911Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:17:46.87283279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:17:46.875843577Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:17:48.568145242Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-07T16:17:49.650682157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:17:49.653781265Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:17:51.247692532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:17:51.251610371Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:17:51.795302314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:51.986124308Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T16:17:54.089264246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:17:54.092288309Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:17:54.255687488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:17:54.259355441Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:17:55.404252408Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
level=info ts=2026-05-07T16:17:56.729850899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:17:56.733018293Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:17:56.795788814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:58.821709539Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-07T16:18:00.766074882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:18:00.769116779Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:18:00.769136757Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:18:00.769157962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:18:00.77172616Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:18:01.795497376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:01.849682425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:18:01.856184836Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:18:02.239904635Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-07T16:18:02.718312043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:18:05.578411929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:18:05.657729956Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=debug ts=2026-05-07T16:18:05.657751205Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-07T16:18:06.585588267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:18:06.588685588Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:18:06.795434235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:18:06.795461538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:18:06.795469272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:18:06.795489947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:08.569150899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:18:08.92535726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:18:08.931691031Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:18:09.075332664Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-07T16:18:11.795038417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:12.492598054Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-07T16:18:12.881829416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:18:12.885100717Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:18:13.046488555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:18:13.05004539Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:18:15.910728783Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=warn ts=2026-05-07T16:18:16.795494469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:17.994752548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:18:17.99913128Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:18:18.911964623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:18:18.915482525Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:18:19.327563002Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-07T16:18:21.74681212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:18:21.749941185Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:18:21.795193976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:21.833340374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:18:21.837801752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:18:21.852850079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:18:21.855828549Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:18:22.745909451Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-07T16:18:26.163783694Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-07T16:18:26.795561652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:28.451268268Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:18:28.839958152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:18:28.843565756Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:18:29.292214163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:18:29.29686849Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:18:29.296889009Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:18:29.296910443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:18:29.300736978Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:18:29.58062399Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T16:18:29.869497581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:18:29.873788954Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:18:30.302447249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:18:30.306035057Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-07T16:18:31.795627083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:32.352077498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:18:32.355590207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:18:32.999266372Z caller=checkpoint.go:372 msg="writing series" size="821 B"
ts=2026-05-07T16:18:34.988231734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50280"
ts=2026-05-07T16:18:35.58018444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:18:36.21599447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:18:36.219190534Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:18:36.416953931Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-07T16:18:36.416978989Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-07T16:18:36.776855963Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:18:36.777926083Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:18:36.795307885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:18:36.795319957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:36.795342362Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:18:36.795353063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:18:39.834667913Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=warn ts=2026-05-07T16:18:41.795491799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:42.005295103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:18:42.009577642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:18:42.009601749Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:18:42.009626866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:18:42.012740771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:18:42.343351929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:18:42.348616095Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:18:43.252412077Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:18:43.886285228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:18:43.889903387Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:18:44.998068979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:18:45.005127194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:18:45.005154987Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:18:45.005197691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:18:45.011960334Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T16:18:46.031502382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46550"
level=debug ts=2026-05-07T16:18:46.670059567Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-07T16:18:46.794805388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:18:49.198881078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:48926"
level=debug ts=2026-05-07T16:18:50.087039734Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=warn ts=2026-05-07T16:18:51.79514347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:52.91286484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:18:52.91455132Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:18:52.979956429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:18:52.983920596Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:18:53.50551072Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-07T16:18:56.776115941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:18:56.783081471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:18:56.795190726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:56.922677508Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-07T16:18:56.922700759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T16:18:58.067087335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:18:58.070042716Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:19:00.34115769Z caller=checkpoint.go:372 msg="writing series" size="480 kB"
level=warn ts=2026-05-07T16:19:01.795397627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:02.877350695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:19:02.880800292Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:19:03.207803024Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:19:03.758818906Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-05-07T16:19:04.60922279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:19:04.614246131Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:19:04.715488615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:19:04.722946757Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T16:19:05.582534659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:19:05.95313312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:19:05.957759895Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:19:06.204536942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:19:06.207378324Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:19:06.430983975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:19:06.434004537Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:19:06.453065253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:19:06.455702086Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:19:06.795056088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:19:06.795088887Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:19:06.795097285Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:19:06.795126424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:07.176481403Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:19:10.309470713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:19:10.312659839Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:19:10.593913836Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=info ts=2026-05-07T16:19:11.485374818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:19:11.488958642Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:19:11.795046308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:12.380908519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:19:12.384757784Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:19:14.01187967Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-07T16:19:15.724254352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:19:15.728282361Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-07T16:19:16.033677652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53606"
level=warn ts=2026-05-07T16:19:16.795457796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:17.429373686Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:19:17.59801337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:19:17.602663857Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:19:18.312685319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:19:18.316308891Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:19:18.54082433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:19:18.544106603Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:19:20.846939536Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=debug ts=2026-05-07T16:19:20.846962464Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-05-07T16:19:21.795695658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:21.825983087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:19:21.830193138Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:19:24.264822515Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-07T16:19:26.795751942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:27.68188743Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-07T16:19:31.099896093Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-07T16:19:31.522626311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:19:31.795232807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:33.604599301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:19:34.519980281Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:19:34.52000135Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
ts=2026-05-07T16:19:34.991800894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51944"
ts=2026-05-07T16:19:35.584213607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:19:35.644329867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:19:35.654172214Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:19:36.776738735Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:19:36.777769925Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:19:36.795152045Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:19:36.79515815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:36.795169225Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:19:36.79517887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:19:37.936139917Z caller=checkpoint.go:372 msg="writing series" size="682 kB"
level=info ts=2026-05-07T16:19:38.768421705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:19:38.772597427Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:19:40.52340453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:19:40.526804965Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:19:41.132337308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:19:41.138999793Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:19:41.353666417Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:19:41.795363416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:44.77060312Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-07T16:19:46.439490645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:19:46.445370681Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:19:46.79578458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:46.844941831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:19:46.848148644Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:19:48.188328357Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-07T16:19:48.324592128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:19:48.327796976Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:19:51.606540997Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:19:51.795352652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:54.26282302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:19:54.267451564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:19:55.023600989Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T16:19:56.794847879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:58.441821791Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-07T16:19:59.827245308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:19:59.832853912Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:20:01.795819889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:01.859285832Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-07T16:20:02.026691788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:20:02.030696717Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:20:02.558532409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:20:02.562615147Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:20:02.562652396Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:20:02.562682444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:20:02.56571433Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:20:03.910342142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:20:03.913334637Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:20:03.913356577Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:20:03.913380893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:20:03.915711522Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:20:05.086844021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:20:05.090080183Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:20:05.2783156Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:20:05.278336779Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=9
ts=2026-05-07T16:20:05.586710776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:20:06.79540961Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:20:06.79543808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:20:06.795447916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:20:06.795491468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:07.802390389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:20:07.807014456Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:20:08.320354957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:20:08.323816279Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:20:08.695585247Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-07T16:20:08.695605857Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-07T16:20:08.782217511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:20:08.785709984Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:20:09.298393433Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:20:09.298450697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:20:09.301756391Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:20:11.474192884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:20:11.477750487Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:20:11.795829581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:12.112240334Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:20:14.914837734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:20:14.918342094Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:20:15.530371097Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:20:16.795617901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:18.947975887Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:20:19.122620578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:20:19.127968851Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:20:20.742126399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:20:20.749129414Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:20:21.386771345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:20:21.389909098Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:20:21.795471148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:22.36604616Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T16:20:24.177184462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:20:25.541772368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:20:25.545311608Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:20:25.783123133Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-07T16:20:26.7955286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:29.201517884Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:20:30.09295015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:20:30.096178427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:20:31.795142521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:32.619484623Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:20:32.692746302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:20:32.695902987Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:20:34.917911593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:34.921724104Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:34.921745194Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:20:34.921767313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:34.924841448Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:20:35.589201623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:20:36.048583954Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-07T16:20:36.048607367Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=8
level=info ts=2026-05-07T16:20:36.578206965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:20:36.582905007Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:20:36.776777004Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:20:36.777841726Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:20:36.79519458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:20:36.795218963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:20:36.795229225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:20:36.795244169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:39.455021093Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T16:20:39.649563898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:20:39.655922837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:20:41.795352054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:42.347604735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:20:42.872401934Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-07T16:20:43.835086203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:20:43.838188617Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:20:45.11161017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:45.115362751Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:20:46.181894432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:20:46.184858519Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:20:46.290595841Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:20:46.290616848Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-07T16:20:46.795015053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:48.377686394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:20:48.382717501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:20:49.707612872Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-07T16:20:50.077947742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:20:50.085833677Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:20:50.84707223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:20:50.850489667Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:20:51.605256141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:20:51.60823472Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:20:51.79517501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:53.126299902Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:20:53.126321369Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-07T16:20:53.48412669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:20:53.487133177Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:20:54.685969376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:20:54.689458466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:20:55.185662126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:20:55.189409322Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:20:55.40337826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:20:55.406245458Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:20:56.543395267Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:20:56.795834411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:59.266272606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:20:59.270981252Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:20:59.961105928Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-07T16:21:01.490680439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:21:01.494464212Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:21:01.795655552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:03.0199854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:21:03.027470804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:21:03.378585223Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-07T16:21:05.591592313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:21:06.795135678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:21:06.795165064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:21:06.795172612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:21:06.795173011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:06.796233353Z caller=checkpoint.go:384 msg="flushing series" totalSize="124 kB" series=3
level=info ts=2026-05-07T16:21:06.803569861Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-07T16:21:06.806760842Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011128972s
level=info ts=2026-05-07T16:21:07.524500239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:21:07.529265651Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:21:11.02645719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:21:11.702286492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:21:11.706093424Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:21:11.795763954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:15.072232967Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:21:15.751965099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:21:15.755499435Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:21:16.040263074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58344"
level=warn ts=2026-05-07T16:21:16.795393261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:21:19.206366684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58650"
level=warn ts=2026-05-07T16:21:21.795288318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T16:21:26.795159611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:26.856304417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:21:26.866733832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:21:27.476959671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:21:27.487004789Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:21:29.61619339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:21:29.619916467Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:21:31.795901309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:34.436031936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:21:34.44140881Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T16:21:35.593688693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:21:35.611675734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:21:35.61507014Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:21:36.777141786Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:21:36.777152036Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:21:36.795486424Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:21:36.795509392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:21:36.795528604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:21:36.795536105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:21:36.7955622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:36.795624295Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-07T16:21:36.795648676Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:21:36.795670336Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.027µs success=true
level=debug ts=2026-05-07T16:21:36.797610931Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-07T16:21:36.797621282Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=1
level=info ts=2026-05-07T16:21:37.323227839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:37.328588993Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:37.328612065Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:21:37.328654499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:37.333027357Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:21:37.365003084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:21:37.368109596Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:21:40.21443263Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:21:40.214454895Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-07T16:21:40.791824987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:21:40.795368142Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:21:41.79521737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:43.662177206Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:21:43.662205055Z caller=checkpoint.go:384 msg="flushing series" totalSize="34 MB" series=1
level=info ts=2026-05-07T16:21:43.991369121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:21:43.999273572Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:21:44.07649782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:21:44.079555193Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:21:45.511102896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:21:45.514723464Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-07T16:21:46.795711731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:47.049707428Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:21:47.943204134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:21:47.94779052Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T16:21:49.2083617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34922"
level=debug ts=2026-05-07T16:21:49.624070232Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:21:49.67026951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:21:49.673552163Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:21:49.987644314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:21:49.991064556Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:21:50.084307971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:21:50.087228396Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:21:50.467035415Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-07T16:21:51.795498913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:52.419242713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:21:52.423297394Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:21:53.885695272Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=info ts=2026-05-07T16:21:55.506919078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:21:55.510061789Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:21:55.71840549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:55.723049939Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:21:56.795463595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:57.303145335Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-07T16:21:57.303172199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-07T16:22:00.44000111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:22:00.443238915Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:22:00.720434573Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-07T16:22:00.822442814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:22:01.795463889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:04.138726166Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=info ts=2026-05-07T16:22:04.84192424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:22:04.845838247Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:22:04.999147226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42016"
ts=2026-05-07T16:22:05.595834663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:22:06.795052432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:22:06.795083019Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:22:06.795070047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:06.795091176Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:22:07.556376653Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-07T16:22:10.973402717Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-07T16:22:11.795843181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:14.391223105Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T16:22:16.795348105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:17.808806618Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-07T16:22:18.034254209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:22:18.042851623Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:22:18.137066757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:22:18.142247233Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:22:18.27083436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:22:18.274195284Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:22:18.928110585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:22:18.931236443Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:22:20.031247291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:22:21.227043041Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:22:21.795283935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:24.000687433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:22:24.008732799Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:22:24.644473357Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-07T16:22:26.102895955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:22:26.106099901Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:22:26.645665504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:22:26.648886835Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:22:26.795811668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:28.062300191Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-07T16:22:28.892514948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:22:28.89783177Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:22:28.897856227Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:22:28.897883026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:22:28.901704263Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:22:28.983686626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:22:28.986814948Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:22:31.480015794Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:22:31.795203812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:32.290563829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:22:32.294979084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:22:34.897929285Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=debug ts=2026-05-07T16:22:34.897951767Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-05-07T16:22:35.598338468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:22:36.776946357Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:22:36.777998062Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:22:36.795387022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:22:36.795426064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:22:36.795436472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:22:36.795524054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:38.281771526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:22:38.285434341Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:22:38.315461611Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:22:38.751983176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:22:38.754779777Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:22:40.718104257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:22:40.719939411Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:22:41.457350003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:22:41.465066373Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:22:41.73286325Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-07T16:22:41.795508981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:42.333076891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:22:42.336117798Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:22:42.491447969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:22:42.495020061Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:22:45.153857868Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T16:22:45.153895598Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-07T16:22:46.765349721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:22:46.768872984Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:22:46.795222522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:47.755801436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:22:47.760257476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:22:47.953968477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:22:47.95735544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:22:48.568341274Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T16:22:51.79536466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:51.986427498Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-07T16:22:52.75027382Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:22:54.291517042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:22:54.295495014Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:22:55.404451634Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=warn ts=2026-05-07T16:22:56.795456351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:58.822316845Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-05-07T16:22:59.305387678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:22:59.310475862Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:23:00.771227775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:23:00.775346423Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:23:01.625696837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:23:01.62866149Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:23:01.795382826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:02.239084556Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=info ts=2026-05-07T16:23:02.681617421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:23:02.68746674Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T16:23:05.600396959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:23:05.657718536Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=debug ts=2026-05-07T16:23:05.657743247Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-07T16:23:05.864165668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:23:05.869318551Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:23:06.795057709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:23:06.795090723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:23:06.795098751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:23:06.795132323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:09.07489051Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-07T16:23:11.795236493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:12.49304086Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-05-07T16:23:15.587363146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:15.590305942Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:15.590328531Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:15.590349059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:15.592754911Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:23:15.910862809Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=warn ts=2026-05-07T16:23:16.794839006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:17.253693936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:23:18.143885747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:23:18.146914879Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:23:18.577417891Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:23:19.21362234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53154"
level=debug ts=2026-05-07T16:23:19.328521122Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-07T16:23:20.333521303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:23:20.340988281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:23:20.341013793Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:20.341043939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:23:20.348014124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:23:21.795699519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:22.228390138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:23:22.231499049Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:23:22.745903467Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:23:25.327084322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:23:25.331942131Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:23:25.993827983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:23:25.997614154Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:23:26.082832585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:23:26.086115339Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:23:26.163351285Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-07T16:23:26.59168606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:23:26.595979974Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:23:26.795086131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:29.581097871Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T16:23:30.552535093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:23:30.565332505Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:23:31.794983705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:32.998967434Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:23:33.79318833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:23:33.796935131Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:23:35.390449389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:35.393547695Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:23:35.529765161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:23:35.533131061Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-07T16:23:35.602246449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:23:36.416754962Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-07T16:23:36.416779187Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-07T16:23:36.777043655Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:23:36.778111462Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:23:36.795443406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:23:36.79547064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:23:36.795462756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:36.795478592Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:23:37.341010849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:23:37.34405667Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:23:37.344086764Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:37.344117875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:23:37.347307997Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:23:38.920432635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:38.924106342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:38.924128522Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:38.924153096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:38.927378544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:23:39.451999705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:23:39.458840273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:23:39.834570824Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=warn ts=2026-05-07T16:23:41.795694546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:42.476120905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:23:42.480026266Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:23:43.251979278Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:23:43.888392926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:23:43.891367603Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:23:43.891389736Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:43.891410094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:23:43.89369664Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:23:46.04908658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38616"
level=debug ts=2026-05-07T16:23:46.669593441Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T16:23:46.795266866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:49.018354872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:23:49.022256059Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-07T16:23:49.216185786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55764"
level=info ts=2026-05-07T16:23:49.963570991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:23:49.96709336Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:23:49.967114166Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:49.96713579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:23:49.969729408Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:23:50.087885185Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:23:51.201893183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:23:51.204919382Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:23:51.79589674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:53.504920708Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=debug ts=2026-05-07T16:23:53.660443573Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:23:55.31701914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:23:55.319565405Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:23:56.795581118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:56.923039809Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-07T16:23:56.92306136Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-07T16:23:57.041650474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:23:58.36940195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:58.372865068Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:24:00.341099037Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=info ts=2026-05-07T16:24:00.593053761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:24:00.596751693Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:24:01.795177728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:02.026996115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:24:02.031508781Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:24:03.758931864Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-07T16:24:04.090052734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:24:04.0944864Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:24:04.33797531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:24:04.344475047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:24:05.448278804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:24:05.452818119Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T16:24:05.60429832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:24:06.79484961Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:24:06.794861173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:06.794894035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:24:06.794904133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:24:07.176685207Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:24:07.310950807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:24:07.315226123Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:24:07.315246828Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:07.315268106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:24:07.318110167Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:24:08.73276926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:24:08.735756915Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:24:10.593846097Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-07T16:24:10.593872691Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-07T16:24:11.186135139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:24:11.189284928Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:24:11.795128344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:11.976692945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:24:11.982063155Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:24:14.012036032Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-07T16:24:15.603601653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:15.607724549Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:24:16.238681799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:24:16.241611438Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:24:16.795653073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:17.428847989Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:24:19.008682703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:24:19.012972604Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-07T16:24:19.218838829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50418"
level=info ts=2026-05-07T16:24:19.57499562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:24:19.578538906Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:24:20.847350081Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=info ts=2026-05-07T16:24:21.392966492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:24:21.396413006Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:24:21.396433709Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:21.396455344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:24:21.399497582Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:24:21.795393449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:24.264460672Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-07T16:24:26.795584928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:26.953985227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:24:27.682809223Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-07T16:24:30.596788511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:24:30.606148304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:24:31.04459712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:24:31.048823108Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:24:31.100304705Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-07T16:24:31.100338773Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-07T16:24:31.743371142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:24:31.746260873Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:24:31.746280883Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:31.746303328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:24:31.748687998Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:24:31.795108243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:34.324241988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:24:34.327760366Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:24:34.51915994Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:24:34.519183808Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-07T16:24:35.004904282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:24:35.006111787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47632"
level=error ts=2026-05-07T16:24:35.007961816Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:24:35.606610926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:24:36.777166882Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:24:36.777210555Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:24:36.795559991Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:24:36.795587119Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:24:36.795595376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:24:36.795623076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:36.982901439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:24:36.986552348Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:24:37.936330524Z caller=checkpoint.go:372 msg="writing series" size="692 kB"
level=info ts=2026-05-07T16:24:39.628086677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:24:39.631851991Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:24:41.353508484Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:24:41.795039642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:44.204419975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:24:44.723856338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:24:44.726942104Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:24:44.771262145Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-07T16:24:46.510778775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:46.513673915Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:46.513696691Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:46.513717417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:46.516123715Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:24:46.795275698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:48.188875632Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-07T16:24:48.686252215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:24:48.689523108Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:24:49.901346655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:24:49.904419596Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:24:50.248315431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:24:50.25292835Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:24:50.334168731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:24:50.340550362Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:24:51.606645792Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:24:51.79528235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:54.930699779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:24:55.024047321Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T16:24:56.795605848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:57.917078436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:57.921590073Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:24:58.159185561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:24:58.16260083Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:24:58.442360331Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-07T16:25:01.084035501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:25:01.089520793Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:25:01.089546903Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:25:01.089575478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:25:01.094493054Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:25:01.795504431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:01.859719769Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-05-07T16:25:02.165428334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:25:02.168733387Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:25:02.397362114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:25:02.40089505Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:25:02.581445969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:25:02.585015048Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:25:05.008167042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42518"
level=debug ts=2026-05-07T16:25:05.278099025Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:25:05.278125221Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=9
ts=2026-05-07T16:25:05.609076667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:25:06.794883533Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:25:06.794894343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:06.794913009Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:25:06.794921097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:25:07.511965687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:25:07.517869631Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:25:08.695789016Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:25:08.695813281Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-07T16:25:10.020268873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:25:10.023542692Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:25:10.791370375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:25:10.794539008Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:25:11.308683438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:25:11.31341903Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:25:11.795113311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:11.986215073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:25:11.993851104Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:25:11.993874932Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:25:11.993908646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:25:12.000451226Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:25:12.112783002Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:25:14.466067448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:25:14.469890939Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:25:15.530829975Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:25:16.207900365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:25:16.211040151Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:25:16.795213505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:18.948582353Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T16:25:19.162468926Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:25:19.222106988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38974"
level=debug ts=2026-05-07T16:25:20.705812124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:25:21.79541178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:22.365630365Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T16:25:25.783888223Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-07T16:25:26.672287609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:25:26.676917287Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:25:26.795580155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:28.59917784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:25:28.606021557Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:25:28.742490269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:25:28.745298516Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:25:28.854665706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:25:28.858251559Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:25:29.0540751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:25:29.057052131Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:25:29.057073454Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:25:29.057095479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:25:29.059492622Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:25:29.200997634Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:25:31.795343107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:32.111532914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:25:32.118752182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:25:32.619158309Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:25:33.471221991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:25:33.474786127Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:25:33.519061209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:25:33.52373999Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T16:25:35.61166963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:25:36.055179657Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-07T16:25:36.055206784Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=8
level=debug ts=2026-05-07T16:25:36.776967458Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:25:36.778012653Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:25:36.795413093Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:25:36.795438967Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:25:36.795447417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:25:36.795487128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:39.454718865Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-07T16:25:40.117567105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:25:40.12462294Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:25:40.663158379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:25:40.667162009Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:25:41.795543668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:42.872671782Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-07T16:25:43.735744183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:25:43.738960644Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-07T16:25:46.054933628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57536"
level=debug ts=2026-05-07T16:25:46.290301954Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:25:46.290323424Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-07T16:25:46.440268038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:25:46.444599132Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:25:46.7952289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:47.806004313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:25:47.806064202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="338 kB" avg_uncomp="338 kB" idle=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:25:47.809241587Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:25:48.098977419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:25:48.102380995Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:25:48.569681106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:25:48.572592909Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-07T16:25:49.224168148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54740"
level=debug ts=2026-05-07T16:25:49.707860346Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-07T16:25:51.508159448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:25:51.515199933Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:25:51.79520962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:52.427484296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:25:52.432156379Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:25:52.702914277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:25:52.705864719Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:25:53.12605033Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:25:53.126071385Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=debug ts=2026-05-07T16:25:56.543084506Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:25:56.795486136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:59.961326824Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T16:26:01.09575357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:26:01.099111332Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:26:01.795284966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:02.871327784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:26:02.875231943Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:26:03.378701794Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:26:04.628368849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:26:04.632593732Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:26:05.012427065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45408"
ts=2026-05-07T16:26:05.613503197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:26:05.638485483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:26:05.641508685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:26:06.794836535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:26:06.79484738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:06.794876171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:26:06.794887191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:26:06.79594505Z caller=checkpoint.go:384 msg="flushing series" totalSize="130 kB" series=3
level=info ts=2026-05-07T16:26:06.803054401Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-07T16:26:06.805116962Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009303641s
level=info ts=2026-05-07T16:26:06.898431534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:26:06.902016789Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:26:07.809360001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:26:07.81282389Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:26:11.544746699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:26:11.549954084Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:26:11.549976738Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:26:11.549997953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:26:11.552643803Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:26:11.795892619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:14.117446919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:26:14.120663544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:26:15.080300082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:26:15.083784891Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:26:16.122825598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:26:16.126335166Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-07T16:26:16.795690381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:17.048814345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:26:18.697148988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:26:18.70222106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:26:21.795617549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:22.767204916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:26:22.768859152Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T16:26:26.795675546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:28.091117934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:26:28.09558097Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:26:29.64282207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:26:29.647574201Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:26:31.795807695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:32.793604611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:32.797271298Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:32.797294217Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:26:32.797315135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:32.799844949Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:26:33.416553442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:26:33.425357461Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:26:34.8136128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:26:34.817056089Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
ts=2026-05-07T16:26:35.615381489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:26:35.910343619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:26:35.913306399Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:26:36.776895229Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:26:36.777951081Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:26:36.795325465Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T16:26:36.795370063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:36.795382732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:26:36.795400849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:26:36.795409437Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:26:36.795481118Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-07T16:26:36.795491105Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:26:36.795507462Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.632µs success=true
level=debug ts=2026-05-07T16:26:36.79739145Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-07T16:26:36.797401346Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=1
level=info ts=2026-05-07T16:26:40.181458804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:26:40.184741636Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:26:40.214308497Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T16:26:40.214328448Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-05-07T16:26:40.491834671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:26:40.495751368Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:26:40.495781529Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:26:40.495811917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:26:40.499397617Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:26:41.757899474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:26:41.761064404Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:26:41.795671336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:42.607366723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:26:42.614456046Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:26:43.661373862Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:26:43.661433358Z caller=checkpoint.go:384 msg="flushing series" totalSize="34 MB" series=1
level=info ts=2026-05-07T16:26:44.755522653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:26:44.758861515Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:26:44.987439056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:26:44.99125115Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:26:44.991272216Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:26:44.991295965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:26:44.994471227Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:26:46.059585447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38858"
level=warn ts=2026-05-07T16:26:46.795722021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:47.049869029Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:26:47.448145763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:26:47.453729522Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:26:47.843001294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:26:47.846060262Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:26:49.100032608Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:26:50.11698299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:50.120287412Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:26:50.466953307Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-07T16:26:51.794847596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:51.90718184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:26:51.91041574Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:26:52.546886678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:26:52.551056948Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:26:53.885876533Z caller=checkpoint.go:372 msg="writing series" size="694 kB"
level=warn ts=2026-05-07T16:26:56.795359304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:57.303613166Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-07T16:26:57.303657816Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=debug ts=2026-05-07T16:27:00.720357315Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-07T16:27:01.199029102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:01.204012972Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:27:01.795714399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:01.941966236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:27:01.945300733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:27:02.186306916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:27:02.190144504Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:27:04.13849439Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-07T16:27:04.976567674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:27:04.980549741Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T16:27:05.016987416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:37738"
level=info ts=2026-05-07T16:27:05.314924712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:27:05.318583511Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:27:05.618113046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:27:06.795098079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:27:06.795126816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:27:06.795134425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:27:06.795123292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:07.140622256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:27:07.144617045Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:27:07.194961219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:07.199898857Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:27:07.556022409Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-07T16:27:10.973786425Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-07T16:27:11.795181209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:14.391194429Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-07T16:27:14.780997657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:27:16.79528429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:17.808969239Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-07T16:27:18.268171664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:27:18.273173441Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:27:19.1850261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:27:19.185071119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="176 kB" avg_comp="176 kB" total_uncomp="887 kB" avg_uncomp="887 kB" idle=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:27:19.188493962Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=debug ts=2026-05-07T16:27:21.226662434Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:27:21.795483237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:22.526757572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:27:22.529889622Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:27:24.644503849Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-07T16:27:25.83491859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:27:25.838594999Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:27:25.947868583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="513 kB" avg_comp="513 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:25.9523071Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:27:26.795204059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:26.924573197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:27:26.927497505Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:27:27.390681702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:27:27.394072037Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:27:28.062093648Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-07T16:27:28.197672921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:27:28.202496618Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:27:29.767771051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:27:29.771333648Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:27:30.003131358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:27:30.006174749Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:27:31.116566223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:27:31.125757442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:27:31.480047139Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:27:31.795199607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:32.188555359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:27:32.192795169Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:27:33.564184408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:27:33.567174085Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:27:33.802117697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:27:33.805216916Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=debug ts=2026-05-07T16:27:34.897680492Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=debug ts=2026-05-07T16:27:34.897701267Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
ts=2026-05-07T16:27:35.619986921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:27:36.776589642Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:27:36.777658613Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-07T16:27:36.795061734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:36.795053802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:27:36.795087373Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:27:36.795097705Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:27:38.315112562Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:27:38.610014381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:27:38.624722377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:27:40.809368711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:27:40.813789583Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:27:41.733331733Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=warn ts=2026-05-07T16:27:41.795790662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:43.186688074Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:27:45.152195904Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T16:27:45.152217136Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=3
level=warn ts=2026-05-07T16:27:46.795685857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:48.568051045Z caller=checkpoint.go:372 msg="writing series" size="939 B"
ts=2026-05-07T16:27:49.230539485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34678"
level=info ts=2026-05-07T16:27:51.080326271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:27:51.085480301Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:27:51.261682927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:27:51.795391828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:51.986358944Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T16:27:52.179682642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:27:52.182618113Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:27:53.699984261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:27:53.703023445Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:27:54.263851188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:27:54.267301963Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=debug ts=2026-05-07T16:27:55.403880213Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=warn ts=2026-05-07T16:27:56.79547419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:57.063396709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:27:57.066987655Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:27:58.525238654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:58.529945205Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:58.529967536Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:27:58.529993164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:58.53413129Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:27:58.821187044Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=info ts=2026-05-07T16:28:00.284981118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:28:00.288437898Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:28:01.795342558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:02.239701976Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
ts=2026-05-07T16:28:05.020914737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33744"
level=info ts=2026-05-07T16:28:05.22543829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:28:05.229190652Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-07T16:28:05.622436351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:28:05.637453195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:28:05.641290824Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:28:05.657846809Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=debug ts=2026-05-07T16:28:05.657863538Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T16:28:06.795279757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:28:06.795315134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:28:06.795326127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:28:06.795355837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:08.462257314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:28:08.462305003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="894 kB" avg_uncomp="894 kB" idle=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:28:08.465895723Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=debug ts=2026-05-07T16:28:09.075219489Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:28:10.95819385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:28:10.963170125Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:28:11.795677673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:12.492826531Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:28:15.321960408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:28:15.324989335Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:28:15.424125353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:28:15.427575251Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:28:15.793596472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:28:15.797092376Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:28:15.910460807Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
ts=2026-05-07T16:28:16.064360583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36712"
level=info ts=2026-05-07T16:28:16.20699853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:28:16.211914116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:28:16.794997133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:19.061159043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:28:19.067808131Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T16:28:19.232445068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:32788"
level=debug ts=2026-05-07T16:28:19.327973765Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-07T16:28:21.229260042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:28:21.232846079Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:28:21.795693525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:22.11855428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:28:22.12360764Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:28:22.369112914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:28:22.372461969Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:28:22.745473962Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:28:22.925105283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:28:22.929021257Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:28:23.389509929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:28:23.392795915Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=info ts=2026-05-07T16:28:23.424859332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:28:23.429236099Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:28:26.163119242Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-07T16:28:26.680524052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:28:26.683719572Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:28:26.795625599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:29.580866412Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-07T16:28:31.795125989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:32.999268285Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:28:33.279233929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:28:33.286563934Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:28:33.660870478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:28:35.624480621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:28:36.417077274Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-07T16:28:36.777186168Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:28:36.77718985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:28:36.795546683Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:28:36.795573464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:28:36.795561465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:36.79558169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:28:37.982343851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:28:39.835054193Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=debug ts=2026-05-07T16:28:39.835076341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=10
level=info ts=2026-05-07T16:28:40.307090244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:28:40.311540426Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:28:41.79508984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:43.252144481Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:28:43.999875724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:28:44.005025131Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=info ts=2026-05-07T16:28:46.294725563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:28:46.300485092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:28:46.669796719Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T16:28:46.795431834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:47.538538137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:28:47.542553533Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:28:49.216147273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:28:49.220154398Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:28:49.646452819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:28:49.649559642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:28:50.087357154Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:28:50.332379278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:28:50.335479327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:28:51.795056752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:53.338228984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:28:53.341480978Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:28:53.505215144Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-07T16:28:54.108356729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:28:54.111874056Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:28:56.087888735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:28:56.09080123Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:28:56.79531656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:56.922790984Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-07T16:29:00.066217781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:29:00.069485588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:29:00.34114219Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=debug ts=2026-05-07T16:29:00.341164737Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-07T16:29:01.795589117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:03.008201234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:29:03.011592562Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:29:03.194145202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:29:03.202863554Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:29:03.463156567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:29:03.466904054Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:29:03.758249624Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-07T16:29:03.987099695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:29:04.192679928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:29:04.195923688Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:29:04.543849132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:29:04.548334117Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:29:04.548352821Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:29:04.548376164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:29:04.552235696Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:29:05.309860885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:29:05.312980273Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:29:05.626800229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:29:05.70794077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:29:05.711789828Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:29:06.795422313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:29:06.795456903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:29:06.795471772Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:29:06.795462596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:07.176372336Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:29:08.704867137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:29:08.708134995Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:29:10.594334688Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=info ts=2026-05-07T16:29:11.410902119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:29:11.414659878Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=warn ts=2026-05-07T16:29:11.795055022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:14.011199156Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-07T16:29:16.068433327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42678"
level=warn ts=2026-05-07T16:29:16.795586121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:17.429004472Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T16:29:20.847171669Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=info ts=2026-05-07T16:29:21.04775722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:29:21.052911403Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:29:21.588660263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:29:21.795201109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:23.041022851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:29:23.045353911Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:29:24.264617217Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=debug ts=2026-05-07T16:29:24.264657902Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-07T16:29:26.795033238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:27.681938045Z caller=checkpoint.go:372 msg="writing series" size="287 kB"
level=debug ts=2026-05-07T16:29:31.099924583Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=warn ts=2026-05-07T16:29:31.795859124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:32.10601312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:29:32.113429382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:29:33.515121669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:29:33.518721079Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:29:34.518347068Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:29:34.518368544Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
ts=2026-05-07T16:29:35.025847596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:49534"
level=info ts=2026-05-07T16:29:35.235605538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:29:35.239450116Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
ts=2026-05-07T16:29:35.629283019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:29:35.863081248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:29:35.867331293Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=debug ts=2026-05-07T16:29:36.465952135Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:29:36.466006097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="204 kB" avg_comp="204 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" idle=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:29:36.472349784Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:29:36.722084007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:36.725228995Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:36.725257631Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:29:36.766311273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="412 kB" avg_uncomp="412 kB" idle=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:29:36.769855874Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:29:36.776705682Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:29:36.777750685Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:29:36.795127772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:29:36.79515324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:29:36.795161122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:29:36.795164224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:37.040259194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:29:37.043364519Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:29:37.066438694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:37.069367115Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:29:37.936197895Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
level=info ts=2026-05-07T16:29:38.109650806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:29:38.114557873Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:29:38.911922597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:29:38.915867987Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:29:39.641762185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:29:39.645504573Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:29:41.352907588Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:29:41.795389407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:44.771092229Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T16:29:45.245486283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:29:45.250134651Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:29:45.818238692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:29:45.821430581Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:29:46.070846263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48336"
level=warn ts=2026-05-07T16:29:46.79535113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:47.606993762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:29:47.610086857Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:29:48.065511501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:29:48.06893188Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:29:48.188454799Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-07T16:29:48.37592489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:29:48.379142583Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:29:51.606061018Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:29:51.758337042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:51.764030131Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:29:51.795490222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:51.860683004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:29:51.864804614Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:29:52.699392805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:29:52.702798289Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:29:53.197372144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:29:53.200507519Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:29:53.200528842Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:29:53.200551877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:29:53.203036526Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:29:55.023724812Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T16:29:56.795527748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:58.442171706Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-07T16:29:59.151370427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:29:59.155296441Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:30:00.740047603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:00.747564554Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:00.747587957Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:00.747624585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:00.754595882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:30:01.795079136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:01.859290234Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-05-07T16:30:02.689153334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:30:02.692490139Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:30:03.482089267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:30:03.68643097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:30:03.689548967Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:30:04.776506254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:30:04.780318623Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:30:05.278525588Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:30:05.27854831Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=9
ts=2026-05-07T16:30:05.631493494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:30:06.795578726Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:30:06.795592554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:06.795614124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:30:06.795628448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:30:07.70932938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:07.712337413Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:07.712360212Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:07.712383856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:07.714940704Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:30:08.695550874Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:30:08.69557589Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-07T16:30:08.730178868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:30:08.733336663Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:30:10.765066609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:30:10.768559803Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=warn ts=2026-05-07T16:30:11.794925416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:12.112459225Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:30:15.262598368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:30:15.268423946Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:30:15.53031827Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:30:16.79570474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:18.59775473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:18.604859405Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:30:18.948653457Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:30:19.469198557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:30:19.472539023Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:30:19.702171814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:30:19.705214453Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:30:20.544596192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:30:20.547789656Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:30:20.547808899Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:20.547829652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:30:20.55049528Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:30:20.865341482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:30:20.867036506Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:30:21.563752799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:30:21.568771513Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:30:21.79489397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:22.029145171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:30:22.033514547Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=debug ts=2026-05-07T16:30:22.365781392Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T16:30:23.109015981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:30:23.112700104Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:30:23.611992169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:30:23.615351779Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:30:23.941316657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:23.944483618Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:30:25.158918303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:30:25.166289134Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:30:25.78404923Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T16:30:26.795260561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:29.201252177Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:30:30.765315517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:30:30.769433724Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:30:30.769457032Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:30.769482547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:30:30.772742532Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-07T16:30:31.79575654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:32.087980809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:30:32.091013526Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:30:32.091035182Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:32.091057732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:30:32.093695072Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:30:32.142897298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:30:32.145775156Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:30:32.618998994Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-07T16:30:33.088610593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:30:33.163666149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:30:33.166772119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:30:33.505193477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:30:33.509813486Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T16:30:35.633552845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:30:35.964439555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:30:35.968552683Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:30:36.057011791Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-07T16:30:36.057039742Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=8
level=debug ts=2026-05-07T16:30:36.776938266Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:30:36.778002311Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:30:36.779310641Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T16:30:36.779350706Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975745 n=1
level=debug ts=2026-05-07T16:30:36.779391599Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975745 n=1
level=info ts=2026-05-07T16:30:36.795368447Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:30:36.79539232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:30:36.795400509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:30:36.795465062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:37.289795673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:30:37.29308891Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:30:37.293113982Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:37.293137958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:30:37.29560044Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:30:38.126961919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:30:38.129999155Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:30:38.472184229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:30:38.475749419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:30:38.475768929Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:38.475793877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:30:38.478842896Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:30:39.454730738Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-07T16:30:41.794920327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:42.871924885Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=debug ts=2026-05-07T16:30:46.290028204Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:30:46.290051539Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-07T16:30:46.59660049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:30:46.60167105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-07T16:30:46.795465773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:48.09469863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:30:48.09794394Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:30:49.006000521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:30:49.009405742Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-07T16:30:49.240693786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50516"
level=debug ts=2026-05-07T16:30:49.708004172Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-07T16:30:51.794903379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:52.223951625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:52.232793082Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:30:52.786611398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:30:52.790362716Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:30:53.125998877Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:30:53.126021505Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-07T16:30:54.137361073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:30:54.140530381Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:30:54.264939541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:54.268123872Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:30:55.523759659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:30:55.527479777Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:30:55.98529741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:30:55.988584452Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:30:56.542915937Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:30:56.795371099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:59.961053381Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-07T16:31:01.150144785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:31:01.153492982Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=warn ts=2026-05-07T16:31:01.795596642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:02.628583188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:31:02.632905065Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:31:03.280967461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:31:03.284405627Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:31:03.378861457Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:31:03.88751099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:31:03.892248758Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
ts=2026-05-07T16:31:05.635731239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:31:06.316936718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:31:06.320041149Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02f1a065%3A19e02f1cb6f%3Ae091f14b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:31:06.320062834Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:31:06.320083437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="19 kB" avg_uncomp="19 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:31:06.322759158Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:31:06.562381239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:31:06.795564819Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:31:06.79559436Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:31:06.795602724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:31:06.795613667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:06.795670959Z caller=checkpoint.go:384 msg="flushing series" totalSize="132 kB" series=3
level=info ts=2026-05-07T16:31:06.803185129Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-07T16:31:06.806776464Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011120996s
level=info ts=2026-05-07T16:31:07.020209056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:31:07.023556626Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:31:07.368279353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:31:07.372764403Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:31:07.712678222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:31:07.716062036Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:31:08.668736711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:31:10.285617255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:31:10.288622446Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:31:10.96123422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:31:10.964369885Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=warn ts=2026-05-07T16:31:11.795099119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:13.265007005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:31:13.268777207Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:31:15.890973906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:31:15.89550924Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:31:16.795508306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:17.128555355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:31:17.132350043Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:31:17.198759032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:31:17.201745539Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:31:18.683853896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:31:18.687451245Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:31:20.176266033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:31:20.179482774Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:31:21.795268369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:23.455142624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:31:23.4633433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:31:24.462143437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:31:24.465401481Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:31:26.79509695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:27.133425689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:31:27.13654212Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:31:29.264683663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:31:29.267824638Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:31:30.766119001Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:31:31.795294869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:34.934388356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:31:34.937783144Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-07T16:31:35.63810325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:31:36.776546452Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:31:36.777608418Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:31:36.794981417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T16:31:36.794995694Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T16:31:36.795011719Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:31:36.795020418Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:31:36.79506181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:36.795202075Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-07T16:31:36.795213871Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:31:36.795237557Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.02µs success=true
level=debug ts=2026-05-07T16:31:36.797966123Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-07T16:31:36.797980652Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=1
level=debug ts=2026-05-07T16:31:40.214748289Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:31:40.214768716Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-07T16:31:41.290469985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:31:41.293945259Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-05-07T16:31:41.79577353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:43.211176232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:31:43.214406056Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:31:43.662536986Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:31:43.662571894Z caller=checkpoint.go:384 msg="flushing series" totalSize="34 MB" series=1
ts=2026-05-07T16:31:46.077277814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59210"
level=info ts=2026-05-07T16:31:46.749195443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:31:46.753050658Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:31:46.795686894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:47.049616716Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:31:48.414593906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:31:48.421929584Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:31:48.506136768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:31:48.509043851Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:31:48.632350174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:31:49.11279484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:31:49.115855642Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:31:50.085842508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:31:50.089521408Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:31:50.466693986Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-07T16:31:50.904531752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:31:50.907671192Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:31:50.943683512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:31:50.946965656Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:31:51.795616539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:53.885516912Z caller=checkpoint.go:372 msg="writing series" size="596 kB"
level=info ts=2026-05-07T16:31:54.501755004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:31:54.50712664Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:31:54.923448503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:31:54.92729693Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:31:56.709449424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:31:56.795086508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:57.302559931Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-07T16:31:57.302589549Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=debug ts=2026-05-07T16:32:00.720320549Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-07T16:32:01.34020221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:32:01.343979922Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=warn ts=2026-05-07T16:32:01.795866385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:01.841004326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:32:01.845723288Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:32:02.454154397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:32:02.45962764Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:32:03.510070518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:32:03.515215972Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:32:03.515239053Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:32:03.515263302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:32:03.520049363Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:32:04.047689356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:32:04.050866498Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:32:04.138453204Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=info ts=2026-05-07T16:32:04.652690691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:32:04.655777399Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:32:04.87941586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:32:04.88252585Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:32:05.197987139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:32:05.201550239Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:32:05.640573671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:32:06.43690818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:32:06.440291994Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=info ts=2026-05-07T16:32:06.795654696Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:32:06.795682923Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:32:06.795671765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:06.795692334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:32:07.555992818Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-07T16:32:10.973525722Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-07T16:32:11.795102005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:12.099473471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:32:12.102922431Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:32:12.102945177Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:32:12.102969164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:32:12.105472036Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:32:13.570458602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:32:13.57591442Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:32:14.391392005Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T16:32:16.79573796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:32:17.810153046Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-07T16:32:17.853922414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:32:17.862443466Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:32:17.862469351Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:32:17.862504318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:32:17.868863594Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T16:32:19.246016122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34666"
level=info ts=2026-05-07T16:32:19.960573837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:32:19.963893689Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:32:21.110329292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:32:21.113597423Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:32:21.225929146Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:32:21.794858137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:32:24.644622478Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-07T16:32:25.698512677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:32:26.718762897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:32:26.795452825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:26.835531027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:32:26.839333593Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:32:28.061895212Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-07T16:32:28.895467195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:32:28.898948358Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:32:31.479259882Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:32:31.573680587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:32:31.576951856Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-05-07T16:32:31.795694536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:32.139951807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:32:32.143724256Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:32:32.143748169Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:32:32.143771373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:32:32.146988947Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:32:34.897576185Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=debug ts=2026-05-07T16:32:34.897599899Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=11
level=info ts=2026-05-07T16:32:35.604430306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:32:35.607703615Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:32:35.642586182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:32:35.74099672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:32:35.747591726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:32:36.002699149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:32:36.010071703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:32:36.777112556Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:32:36.77752282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:32:36.794839321Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:32:36.794840606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:32:36.79486302Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:32:36.794883235Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:32:38.314725066Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=debug ts=2026-05-07T16:32:41.732809351Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=warn ts=2026-05-07T16:32:41.795231013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:42.708115329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:32:42.712182827Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:32:43.349961081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:32:43.355258976Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:32:45.127864082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:32:45.131444734Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:32:45.152469084Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-07T16:32:45.152486953Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-07T16:32:45.310653585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:32:45.315510455Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:32:45.443998853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:32:45.447037211Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:32:46.795273398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:47.283970525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:32:47.287017848Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:32:48.567696419Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-07T16:32:48.728183058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:32:48.73379765Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:32:50.323493013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:32:50.326921842Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:32:51.795237218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:32:51.986069721Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T16:32:53.579262451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:32:53.582685186Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:32:54.035988377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:32:54.040313883Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:32:55.193876413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:32:55.198848102Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:32:55.29298897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:32:55.296108093Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:32:55.403252357Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=info ts=2026-05-07T16:32:55.656591493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:32:55.661219478Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:32:55.787834693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:32:55.7908129Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:32:55.790833989Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:32:55.79085758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:32:55.793322553Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:32:56.795315908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:57.971697523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:32:57.975132887Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=debug ts=2026-05-07T16:32:58.367227704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:32:58.821399241Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-07T16:33:00.17068433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:33:00.17426016Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:33:00.962199926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:33:00.965152869Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:33:01.316218379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:33:01.322947802Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:33:01.795410639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:33:02.239110157Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-07T16:33:03.387714035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:33:03.390723931Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:33:03.390744207Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:33:03.390765267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:33:03.393100068Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:33:03.589909155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:33:03.59347991Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:33:05.644700649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:33:05.656913617Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=debug ts=2026-05-07T16:33:05.65693671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-07T16:33:06.456011231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:33:06.459314417Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:33:06.795664373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:33:06.795693275Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:33:06.79570132Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:33:06.795717621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:33:09.074400784Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:33:10.553378539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:33:10.557253501Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:33:10.55727637Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:33:10.557303748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:33:10.560555962Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:33:11.795363267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:12.160544986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:33:12.164238751Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:33:12.250824317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:33:12.491843527Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-05-07T16:33:15.149066025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:33:15.152103499Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:33:15.909691204Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
ts=2026-05-07T16:33:16.081910899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59556"
level=warn ts=2026-05-07T16:33:16.795665196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:17.693094818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:33:17.696593542Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:33:19.327192693Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-07T16:33:21.795445963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:33:22.745128656Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:33:23.276957164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:33:23.281101782Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:33:25.732820015Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:33:26.163097528Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-07T16:33:26.795383814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:28.285702233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:33:28.291002649Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:33:29.580879789Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-07T16:33:30.481604889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:33:30.486048428Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:33:31.795025587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:31.935408505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:33:31.942154815Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:33:32.998535954Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:33:34.593686385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:33:34.597845872Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-07T16:33:35.646505328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:33:35.937411222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:33:35.942452822Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:33:36.416204385Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-07T16:33:36.416226665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-07T16:33:36.777121446Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:33:36.777427826Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:33:36.794756822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:33:36.794784729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:33:36.79479271Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:33:36.794871342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:39.225489329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:33:39.229107655Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:33:39.229128775Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:33:39.229150669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:33:39.23201811Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:33:39.834196545Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=warn ts=2026-05-07T16:33:41.795120189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:42.808143681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:33:42.812971115Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:33:42.851035838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:33:42.853867577Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:33:43.252145733Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:33:43.867344752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:33:43.870939284Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:33:44.19094305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:33:44.194035533Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:33:44.546016983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:33:44.554266989Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T16:33:46.08453566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57322"
level=debug ts=2026-05-07T16:33:46.669913695Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T16:33:46.79566423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:47.096620237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:33:47.099784825Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:33:47.153896378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:33:47.15698727Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:33:47.556007008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:33:47.559091546Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:33:48.456742219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:33:48.461441977Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:33:48.570909285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:33:48.574159446Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=debug ts=2026-05-07T16:33:50.086872354Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:33:50.919917291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:33:50.923486641Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:33:51.795875536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:33:51.842031598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:33:51.845786622Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:33:51.845808258Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:33:51.845829085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:33:51.8483997Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:33:52.681553123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:33:52.685146207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:33:53.504763633Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-07T16:33:55.14696804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:33:55.150257703Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=debug ts=2026-05-07T16:33:55.274184336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:33:55.275693603Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:33:56.687562712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:33:56.69038927Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:33:56.718452364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:33:56.721293481Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:33:56.794800512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:33:56.922349694Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=debug ts=2026-05-07T16:33:56.922371068Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-07T16:33:57.084222111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:33:57.087220214Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:33:58.281846471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:33:58.284789026Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:33:58.739432876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:33:58.744282147Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:33:59.552302462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:34:00.340268244Z caller=checkpoint.go:372 msg="writing series" size="493 kB"
level=warn ts=2026-05-07T16:34:01.795728146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:03.758051188Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
ts=2026-05-07T16:34:05.649012128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:34:06.795762649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:34:06.795793549Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:34:06.79580159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:34:06.795825281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:07.175377789Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:34:07.235547368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:34:07.239630487Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:34:07.635848373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:34:07.639089864Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:34:10.594198723Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=warn ts=2026-05-07T16:34:11.795385034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:14.011217463Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-07T16:34:14.01124168Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-07T16:34:16.52565892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:34:16.531002083Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:34:16.795406982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:17.429194572Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T16:34:20.847354684Z caller=checkpoint.go:372 msg="writing series" size="383 kB"
level=info ts=2026-05-07T16:34:21.639163658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:34:21.647468073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:34:21.795176508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:24.264279355Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=debug ts=2026-05-07T16:34:25.45878341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:34:26.795687315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:34:27.562342316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:34:27.567348169Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:34:27.682113517Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
level=info ts=2026-05-07T16:34:27.767015548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:34:27.772646195Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:34:29.767315412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:34:29.77554128Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:34:30.16415427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:34:30.167612233Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=debug ts=2026-05-07T16:34:31.100065429Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-07T16:34:31.100087342Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-07T16:34:31.289163277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:34:31.291552517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:34:31.293510045Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:34:31.293527749Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:34:31.402974268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:34:31.405995806Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:34:31.795716914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:34:31.990688637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:34:31.995304726Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:34:31.995329869Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:34:31.99535443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:34:31.99997041Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:34:34.518433857Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:34:34.518464706Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-07T16:34:35.042435071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51790"
ts=2026-05-07T16:34:35.651593704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:34:36.77692729Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:34:36.777969694Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:34:36.795331422Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:34:36.795368213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:34:36.795376702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:34:36.795395286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:37.645682884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:34:37.935923316Z caller=checkpoint.go:372 msg="writing series" size="505 kB"
level=info ts=2026-05-07T16:34:38.831074376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:34:38.834097114Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:34:40.870193911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:34:40.878158312Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:34:41.274204849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:34:41.278247969Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:34:41.353388531Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:34:41.795130371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:34:42.05847216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:34:42.061800202Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:34:42.484592545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:34:42.487664318Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:34:42.754239935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:34:42.757657085Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:34:42.94125752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:34:42.944463327Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:34:44.770749395Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T16:34:45.240984977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f3b04089b1c7727e immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:34:45.24489354Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f3b04089b1c7727e/19e0305d81f%3A19e030d742a%3A74b44b87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-qgor\"}"
level=error ts=2026-05-07T16:34:45.244915482Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:34:45.244938955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:34:45.248198311Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:34:45.692534032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:34:45.697336321Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:34:46.794894243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:34:47.084263953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:34:47.087557548Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:34:47.147539957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:34:47.150437621Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:34:47.586470023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:34:47.591682136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:34:48.18867552Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:34:51.606416003Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:34:51.795131621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:34:51.834283289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:34:51.838866751Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:34:55.024040374Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-07T16:34:56.017820659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:34:56.020982156Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:34:56.795462185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:34:56.916827106Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:34:57.586373508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:34:57.591166355Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:34:58.426609843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:34:58.430237314Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:34:58.441369979Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-05-07T16:35:01.795269265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:35:01.85942021Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
ts=2026-05-07T16:35:05.044398178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56386"
level=debug ts=2026-05-07T16:35:05.278123051Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:35:05.278145105Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=9
ts=2026-05-07T16:35:05.65417618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:35:06.795770129Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:35:06.795799573Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:35:06.795807331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:35:06.795844644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:35:08.630823139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:35:08.641167428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:35:08.695653488Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-07T16:35:08.695678008Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-07T16:35:11.093168467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:35:11.096628882Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:35:11.794869289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:35:12.11237117Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:35:13.548375072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:35:13.553049125Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:35:14.011827689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:35:14.015065518Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:35:15.529982334Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-07T16:35:16.090228111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52832"
level=info ts=2026-05-07T16:35:16.381685493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:35:16.384815438Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:35:16.795325262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:35:17.564421055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:35:17.569390605Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:35:18.679973867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:35:18.683606456Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-07T16:35:18.948395228Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:35:19.882352453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:35:19.88578224Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:35:20.102327155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:35:20.10570123Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:35:20.202059808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:35:20.205536759Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:35:21.794905901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:35:22.190223042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:35:22.193195036Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:35:22.365567346Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T16:35:22.87702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:35:25.783835646Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-07T16:35:25.99746198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:35:26.000930007Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:35:26.396111929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:35:26.399947216Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:35:26.795225553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:35:26.820374736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:35:26.824807683Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:35:26.824825767Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:35:26.824850085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:35:26.829060393Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:35:29.200976075Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:35:30.03884188Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:35:31.312857229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:35:31.316405464Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:35:31.344421553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:35:31.347897431Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:35:31.494485093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:35:31.50130693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:35:31.795258515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:35:32.619289781Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:35:32.674498203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:35:32.677506065Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-07T16:35:35.656528969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:35:35.887321587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:35:35.890737812Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:35:35.916774462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:35:35.919591835Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:35:36.055354842Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-07T16:35:36.055377309Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=8
level=info ts=2026-05-07T16:35:36.640747531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:35:36.644720702Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:35:36.726084457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:35:36.729183031Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:35:36.777132608Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:35:36.777517502Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:35:36.794837177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:35:36.794842331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:35:36.794863604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:35:36.794873616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:35:38.882337619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:35:38.88780148Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:35:39.454476309Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-07T16:35:41.795132193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:35:42.871677486Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
ts=2026-05-07T16:35:46.092316605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53648"
level=debug ts=2026-05-07T16:35:46.290726708Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:35:46.290749841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-05-07T16:35:46.794823406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:35:49.257324644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55656"
level=debug ts=2026-05-07T16:35:49.707056925Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-07T16:35:51.606271121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:35:51.615269117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:35:51.795194393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:35:53.125788579Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:35:53.125811507Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-07T16:35:54.637754772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:35:54.641610289Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:35:54.67381691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:35:54.677021218Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:35:55.600316658Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:35:56.54282284Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-07T16:35:56.628382908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:35:56.631972024Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:35:56.699459271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:35:56.702832481Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:35:56.795388029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:35:57.702682454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:35:57.707548488Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:35:59.960163153Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-07T16:36:00.937536502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:36:00.942154086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:36:01.795895383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:02.47235319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:36:02.476963295Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:36:02.746623917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:36:02.749769583Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:36:03.378806706Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:36:03.52856184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:36:03.532856071Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:36:04.188969388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:36:04.19219832Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-07T16:36:05.658427673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:36:06.795394188Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:36:06.795421798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:36:06.795430609Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:36:06.795450619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:36:06.795499796Z caller=checkpoint.go:384 msg="flushing series" totalSize="138 kB" series=3
level=info ts=2026-05-07T16:36:06.802375242Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-07T16:36:06.804850553Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009433031s
level=info ts=2026-05-07T16:36:08.172231595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:36:08.176102982Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:36:08.180135936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:36:08.183316954Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:36:08.376971624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:36:08.380751278Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:36:11.795879771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:11.957204452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:36:11.963557199Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:36:12.252676247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:36:12.256020308Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:36:12.256039728Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:12.256061978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:36:12.259195439Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:36:13.115379176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:36:13.118720316Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:36:13.118739262Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:13.118761076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:36:13.121621573Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:36:14.014187791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:36:14.018271846Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
ts=2026-05-07T16:36:16.094236939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56774"
level=warn ts=2026-05-07T16:36:16.795071454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:16.931430926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:36:16.935139999Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:36:18.16326685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:36:18.167164191Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:36:18.167187501Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:18.167208974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:36:18.169775648Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:36:19.52759117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:36:21.795163425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:36:21.906626355Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:36:23.387501041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:36:23.390603545Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:36:24.031937858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:36:24.039571359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:36:24.442575226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:36:24.445705413Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:36:24.445726587Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:24.445748922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:36:24.448062535Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:36:24.831336067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:36:24.835778196Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:36:26.795586386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:28.055439738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:36:28.05880068Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:36:31.795593521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:32.917453227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:36:32.922298556Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:36:35.088307424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:36:35.09188336Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-07T16:36:35.660246646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:36:36.776756891Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:36:36.777794972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:36:36.777829032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:36:36.781522837Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:36:36.781537931Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:36.781561635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:36:36.784264648Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:36:36.795407728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:36:36.795427522Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:36:36.795422557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:36.7954357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:36:36.795435419Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:36:36.795576574Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-07T16:36:36.795584582Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:36:36.795602773Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.692µs success=true
level=debug ts=2026-05-07T16:36:36.798541828Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-07T16:36:36.798551872Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=1
level=info ts=2026-05-07T16:36:37.392404022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:36:37.396261258Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:36:37.990321887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:36:37.993800514Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:36:38.758596433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:36:38.765975259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:36:38.765997084Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:38.766025317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:36:38.773020557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:36:40.214328677Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:36:40.214351864Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-07T16:36:41.795030335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:42.218932399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:36:42.222774035Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:36:42.222795699Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:42.222817539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:36:42.225534342Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:36:42.257622778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:36:42.262589592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:36:43.662333309Z caller=checkpoint.go:372 msg="writing series" size="34 MB"
level=debug ts=2026-05-07T16:36:43.662360761Z caller=checkpoint.go:384 msg="flushing series" totalSize="34 MB" series=1
level=info ts=2026-05-07T16:36:44.693776906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:36:44.697206759Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:36:46.206995446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:36:46.211209624Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:36:46.795410911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:36:47.049279685Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:36:50.434041835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:36:50.44523798Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:36:50.467467172Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-07T16:36:51.08922283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:36:51.092554653Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:36:51.795754051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:52.306013181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:36:52.309075417Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:36:52.353528257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:36:52.357784339Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:36:52.936344366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:36:52.939707431Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:36:53.345819083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:36:53.348661215Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:36:53.514770851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:36:53.518224188Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:36:53.518246522Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:36:53.518274644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:36:53.5211413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:36:53.885679151Z caller=checkpoint.go:372 msg="writing series" size="690 kB"
level=debug ts=2026-05-07T16:36:55.750044402Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:36:56.140248675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:36:56.143882153Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:36:56.699070346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:36:56.702310698Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=warn ts=2026-05-07T16:36:56.794825472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:36:56.838019701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:36:56.841153852Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:36:57.302711778Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-07T16:36:57.302733844Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-07T16:36:57.690399924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:36:57.694135538Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:36:57.848692406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:36:57.856085509Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:37:00.719940172Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-07T16:37:01.795076622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:02.721118441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:37:02.724481105Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:37:04.13818023Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T16:37:04.55138991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:37:04.555070267Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:37:05.051779053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:49700"
level=info ts=2026-05-07T16:37:05.266457388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:37:05.270107978Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:37:05.270129165Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:37:05.270152536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:37:05.273065736Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:37:05.349743504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:37:05.353460346Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:37:05.662788184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:37:05.851493486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:37:06.279845618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:37:06.283078605Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:37:06.794768267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:37:06.79480281Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:37:06.794811636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:37:06.794807379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:37:07.556220841Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-07T16:37:08.271457998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:37:08.274478293Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:37:08.274498791Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:37:08.274519509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:37:08.276768793Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:37:10.712786828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:37:10.716034042Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:37:10.973280657Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-07T16:37:11.795594496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:37:14.39097995Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T16:37:16.795027146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:37:17.809657428Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-07T16:37:18.274970809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:37:18.283948883Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:37:20.820697739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:37:20.825152635Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:37:21.226226041Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:37:21.795401611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:23.999684095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:37:24.003456032Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:37:24.300188378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:37:24.304189985Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:37:24.626106854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:37:24.629134497Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:37:24.629151581Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:37:24.629171193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:37:24.633332455Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:37:24.644730298Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T16:37:24.655677328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:37:24.658546838Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-05-07T16:37:26.795686132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:26.970166484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:37:26.975372318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:37:27.204475297Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:37:28.061980245Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-07T16:37:29.626362475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:37:29.630208965Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:37:29.705522274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:37:29.710342132Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:37:31.479827067Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-07T16:37:31.795081723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:32.05015579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:37:32.056585609Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:37:34.897521273Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
ts=2026-05-07T16:37:35.665182297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:37:36.714142304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:37:36.720548248Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:37:36.776716792Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:37:36.77776909Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:37:36.795152333Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:37:36.795179312Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:37:36.795187253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:37:36.795226338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:36.863478558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:37:36.867251765Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:37:38.164155357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:37:38.167077862Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:37:38.31549685Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:37:38.537134482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:37:38.540182019Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:37:41.500980488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:37:41.504362458Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=debug ts=2026-05-07T16:37:41.733201508Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=debug ts=2026-05-07T16:37:41.733223378Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=13
level=warn ts=2026-05-07T16:37:41.795377013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:42.929995739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:37:42.933724139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:37:43.11221552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:37:43.115399357Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:37:43.216758464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:37:43.220462476Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:37:43.665510271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:37:43.668740482Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:37:43.668761075Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:37:43.668783956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:37:43.671293715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:37:44.811139103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:37:44.814415245Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:37:45.152655705Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T16:37:45.152677049Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=warn ts=2026-05-07T16:37:46.795273676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:48.222577637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:37:48.226696962Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-07T16:37:48.568758129Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-07T16:37:51.79547821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:51.847595266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:37:51.851926463Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:37:51.986662877Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-07T16:37:52.135958618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:37:52.139150643Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:37:54.163168178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:37:54.166473336Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:37:55.403990915Z caller=checkpoint.go:372 msg="writing series" size="422 kB"
level=warn ts=2026-05-07T16:37:56.795844168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:37:58.130135459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:37:58.13419516Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:37:58.821312719Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-07T16:37:59.129931841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:37:59.132893092Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:38:00.800587851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:38:00.802127567Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T16:38:01.773012227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:38:01.795260358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:01.918602337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:38:01.921492398Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:38:02.234020953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:38:02.237977083Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:38:02.2391414Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-07T16:38:05.657147021Z caller=checkpoint.go:372 msg="writing series" size="656 kB"
level=debug ts=2026-05-07T16:38:05.657170639Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
ts=2026-05-07T16:38:05.667451487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:38:06.795195049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:38:06.795223572Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:38:06.795231501Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:38:06.795243948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:38:09.07536589Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:38:11.40049129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:38:11.405883017Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:38:11.795317061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:12.390993116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:38:12.399065188Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:38:12.492433571Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-07T16:38:13.425344607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:38:13.432816791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:38:13.528229885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:38:13.531342674Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:38:13.843414711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:38:13.846364386Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:38:14.621905128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:38:14.624986674Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:38:15.910298079Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-05-07T16:38:16.79518633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:17.797684299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:38:17.800936997Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:38:18.36525893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:38:18.368310965Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:38:19.327951623Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-07T16:38:19.622892848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:38:19.627121786Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:38:21.795832996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:38:22.745299529Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:38:24.607516221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:38:24.613963086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:38:26.163037105Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-07T16:38:26.257394059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:38:26.260978502Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:38:26.795271011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:27.938288574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:38:27.943045968Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:38:27.943067319Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:38:27.943091065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:38:27.947216724Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:38:28.518916862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:38:28.52258448Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:38:28.750389518Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:38:29.580873063Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-07T16:38:29.622969137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:38:29.626819376Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:38:30.007121455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:38:30.010456472Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:38:31.795765678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:38:32.998779342Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=info ts=2026-05-07T16:38:33.505682293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:38:33.508766619Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:38:34.353983378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:38:34.358008339Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:38:34.545609804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:38:34.548941704Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:38:34.548961788Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:38:34.548983804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:38:34.551601293Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
ts=2026-05-07T16:38:35.669609646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=debug ts=2026-05-07T16:38:36.417257888Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=debug ts=2026-05-07T16:38:36.417289431Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-07T16:38:36.77731524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:38:36.777318943Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:38:36.795679725Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:38:36.795705236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:38:36.795712837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:38:36.795729752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:39.769984725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:38:39.773373957Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:38:39.834967325Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=info ts=2026-05-07T16:38:40.031217101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:38:40.036064343Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:38:41.617688614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:38:41.622169824Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:38:41.795088232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:41.989401123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:38:41.992347577Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:38:42.8912556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:38:43.252390562Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:38:46.535868157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:38:46.539136829Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:38:46.669825921Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=warn ts=2026-05-07T16:38:46.795669385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:47.909986644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:38:47.912870161Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:38:47.912892088Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:38:47.912915746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:38:47.915142821Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-07T16:38:49.268580002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55988"
level=info ts=2026-05-07T16:38:49.464567549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:38:49.46856285Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:38:50.087576183Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=warn ts=2026-05-07T16:38:51.795201324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:38:52.653324035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:38:52.661455451Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:38:52.661479221Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:38:52.661514162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:38:52.669049402Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:38:53.505236169Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:38:54.634013207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:38:54.639648627Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:38:56.795198958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:38:56.922828836Z caller=checkpoint.go:372 msg="writing series" size="287 kB"
level=debug ts=2026-05-07T16:38:56.922849778Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-07T16:38:58.20922613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:38:58.21265478Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:38:58.212672418Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:38:58.212692328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:38:58.215238626Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:38:59.078618628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:38:59.082464269Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-07T16:38:59.484914421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:38:59.488157667Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:39:00.34170329Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=info ts=2026-05-07T16:39:01.058251205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:39:01.061737171Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:39:01.795164808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:03.558845894Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:39:03.759075772Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-07T16:39:03.89255592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:39:03.896070605Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-07T16:39:05.671957661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:39:06.795575805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:39:06.795604801Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:39:06.795613265Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:39:06.795621124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:07.176430011Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:39:08.89226156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:39:08.895895176Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:39:10.25175962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:39:10.256961976Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:39:10.593826404Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-07T16:39:10.593849624Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-07T16:39:10.726932855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:39:10.735359182Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:39:11.150603335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:39:11.153884764Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:39:11.795818363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:39:12.429040738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:39:12.432485475Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:39:12.664066921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:39:12.667030438Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:39:13.295959756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:39:13.30425685Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:39:13.627306959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:39:13.632708836Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:39:14.011778124Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-07T16:39:14.029790703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:39:14.034535439Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:39:14.444379385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:39:14.448032989Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=warn ts=2026-05-07T16:39:16.795471282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:17.429645218Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:39:19.743690521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:39:19.747041327Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:39:19.747061707Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:39:19.74708427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:39:19.749804477Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:39:20.243537216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:39:20.24755961Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:39:20.847560716Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
level=warn ts=2026-05-07T16:39:21.795400787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:39:22.802672779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:39:22.806156826Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:39:23.601093534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:39:23.605149354Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:39:24.265084459Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-07T16:39:26.116213511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:39:26.121204698Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:39:26.144266658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:39:26.147188862Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:39:26.79548852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:27.682521146Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=info ts=2026-05-07T16:39:28.864910258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:39:28.869054486Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:39:29.247687664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:39:29.251427319Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:39:29.415037082Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:39:30.833466601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:39:30.837579781Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:39:30.83760066Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:39:30.837624837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:39:30.840990365Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:39:31.100319781Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-07T16:39:31.100341672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-05-07T16:39:31.795324356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:39:33.99768392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:39:34.001344904Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:39:34.518981148Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:39:34.519003993Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-07T16:39:35.67472945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:39:36.777335564Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:39:36.777347238Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:39:36.795757298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:39:36.79578696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:39:36.795794742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:39:36.795807426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:37.935672235Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=info ts=2026-05-07T16:39:38.649991007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:39:38.653350166Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:39:39.646352436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:39:39.654574356Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:39:40.976394819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:39:40.979992561Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:39:41.352855092Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-07T16:39:41.795305923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:43.004706607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:39:44.771235103Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-07T16:39:44.814388028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:39:44.818702038Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:39:46.106411475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33076"
level=info ts=2026-05-07T16:39:46.329591959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:39:46.3327876Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:39:46.795245397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:48.188560996Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-07T16:39:49.325864196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:39:49.330613081Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:39:51.605973211Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:39:51.795861916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:39:53.565349807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:39:53.568855419Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:39:53.683176458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:39:53.686725976Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:39:54.961733515Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:39:55.023698235Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-05-07T16:39:56.795126176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:39:58.442025814Z caller=checkpoint.go:372 msg="writing series" size="6.4 kB"
level=info ts=2026-05-07T16:40:00.611539189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:40:00.616473431Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:40:01.567122947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:40:01.580191384Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:40:01.795016397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:01.859162264Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-05-07T16:40:02.531766057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:40:02.53778729Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:40:04.374998807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:40:04.378482338Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:40:04.387512013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:40:04.390282737Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:40:05.278209921Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:40:05.278233033Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=9
ts=2026-05-07T16:40:05.677161015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:40:05.974285069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:40:05.977536686Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:40:06.329256879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:40:06.33460356Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:40:06.795433783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:40:06.795455483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:06.79547028Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:40:06.795481693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:40:07.845316216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:40:07.850662964Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:40:08.695698467Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:40:08.695721213Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T16:40:10.541450055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:40:10.544671268Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:40:11.157354747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:40:11.160907972Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:40:11.160931536Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:40:11.16095301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:40:11.163648355Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=warn ts=2026-05-07T16:40:11.795436088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:40:12.00968638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:40:12.012794493Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:40:12.113050742Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:40:12.72882078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:40:12.731865165Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:40:15.531005269Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:40:16.795308977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:40:17.785341321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:40:17.78978703Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:40:17.789809807Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:40:17.789835412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:40:17.793182092Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:40:18.947986291Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:40:20.717662044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:40:20.722080613Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:40:20.803790729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:40:20.807902563Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:40:21.557821711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:40:21.560980242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:40:21.795140877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:22.365717945Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T16:40:24.898397279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:40:24.90220642Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=info ts=2026-05-07T16:40:25.095866054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:40:25.102812039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:40:25.256674448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:40:25.260152903Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:40:25.63370392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:40:25.637384272Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:40:25.784216223Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-07T16:40:26.795049877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:40:26.969467594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:40:26.973328081Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:40:28.540417382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:40:29.201397656Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:40:29.520364906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:40:29.528352726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:40:29.53202671Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:40:31.795406866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:32.619576594Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-07T16:40:35.062556757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60178"
ts=2026-05-07T16:40:35.679886566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=debug ts=2026-05-07T16:40:36.055151889Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-07T16:40:36.055177764Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=8
level=info ts=2026-05-07T16:40:36.777160493Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:40:36.777627082Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:40:36.795007326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:40:36.795005873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:36.795038077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:40:36.795046879Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:40:39.454624655Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-07T16:40:40.855328979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:40:40.860987612Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:40:41.795231653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:42.872343587Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-07T16:40:44.293354522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:40:44.296715877Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:40:45.449362469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="944 kB" avg_comp="944 kB" total_uncomp="7.5 MB" avg_uncomp="7.5 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:40:45.454813895Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T16:40:46.109835315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57970"
level=debug ts=2026-05-07T16:40:46.29045186Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-07T16:40:46.290475075Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-05-07T16:40:46.794910888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:40:49.708062858Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-07T16:40:50.46520849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:40:50.482681628Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:40:51.795378016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:40:52.046484751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:40:52.05596218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:40:52.736256048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:40:52.739437406Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=debug ts=2026-05-07T16:40:53.126756644Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:40:53.126783393Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=debug ts=2026-05-07T16:40:53.970550531Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:40:54.395823698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:40:54.398927861Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:40:55.028788749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:40:55.031873616Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:40:55.35337504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:40:55.356399791Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:40:56.542990885Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:40:56.794813014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:40:57.752756635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:40:57.756232326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:40:59.96072481Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-07T16:41:00.603789003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:41:00.607235502Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:41:01.396971461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:41:01.401888371Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:41:01.401910371Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:41:01.401936728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:41:01.406208111Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:41:01.795818879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:41:01.881103788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:41:01.884997862Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:41:02.392393786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:41:02.396034605Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:41:03.160157949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:41:03.164056887Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:41:03.37902875Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-07T16:41:04.33379451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:41:04.338664509Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:41:05.275799197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:41:05.278897047Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-05-07T16:41:05.682561586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:41:06.79564799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:41:06.795680607Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:41:06.795693485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:41:06.795681444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:41:06.796751654Z caller=checkpoint.go:384 msg="flushing series" totalSize="144 kB" series=3
level=info ts=2026-05-07T16:41:06.802964246Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000019.tmp new=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-07T16:41:06.804056753Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008311918s
level=info ts=2026-05-07T16:41:07.169758181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:41:07.172908557Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:41:09.146272951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:41:09.720012737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:41:09.723169797Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:41:10.435288961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:41:10.438824993Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:41:11.795233557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:41:13.280166243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:41:13.284466549Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-07T16:41:13.2844872Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:41:13.284508242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:41:13.287560207Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:41:13.356685463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:41:13.361273903Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:41:15.76257383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:41:15.765932971Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=warn ts=2026-05-07T16:41:16.794883431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:41:18.113179018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:41:18.117199945Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-07T16:41:19.276580006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40272"
level=info ts=2026-05-07T16:41:21.110841223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:41:21.119075969Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:41:21.795543545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:41:25.016920911Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:41:25.397401647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:41:25.401748018Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-07T16:41:26.795115846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T16:41:31.795305308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:41:35.685259506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:41:36.559768547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:41:36.565365258Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:41:36.566326779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:41:36.570752411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:41:36.776706294Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:41:36.777757539Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:41:36.795129583Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:41:36.795150092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:41:36.795173718Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:41:36.79518196Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:41:36.795219458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:41:36.795283Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000020
level=info ts=2026-05-07T16:41:36.795291093Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:41:36.795314173Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.83µs success=true
level=debug ts=2026-05-07T16:41:36.797079599Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-07T16:41:36.797090223Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=1
level=info ts=2026-05-07T16:41:36.892608482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:41:36.896114696Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=info ts=2026-05-07T16:41:38.66768539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:41:38.670873692Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:41:38.885563708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:41:38.889028508Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:41:40.172070713Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:41:40.172095043Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-07T16:41:40.670065732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:41:40.673292552Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:41:41.265017654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:41:41.268293619Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:41:41.744378393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:41:41.747927542Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=warn ts=2026-05-07T16:41:41.795452152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:41:43.589514666Z caller=checkpoint.go:372 msg="writing series" size="35 MB"
level=debug ts=2026-05-07T16:41:43.589551499Z caller=checkpoint.go:384 msg="flushing series" totalSize="35 MB" series=1
level=info ts=2026-05-07T16:41:44.942814492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:41:44.94847033Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:41:44.948500398Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:41:44.948526306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:41:44.952826121Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T16:41:46.113525449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48418"
level=warn ts=2026-05-07T16:41:46.794987722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:41:46.921343696Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:41:47.052376485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:41:47.056799331Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:41:49.026700016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="144 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:41:49.033374501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:41:50.150556955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:41:50.153133713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:41:50.15398057Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:41:50.15592757Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:41:50.29569172Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-07T16:41:51.795347581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:41:51.98426975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:41:51.990293891Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:41:52.333528318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:41:53.671434515Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=info ts=2026-05-07T16:41:55.427526752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:41:55.433604355Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:41:56.36014243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:41:56.363737132Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-07T16:41:56.795372168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:41:57.046990426Z caller=checkpoint.go:372 msg="writing series" size="846 kB"
level=debug ts=2026-05-07T16:41:57.047016426Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-07T16:41:58.064482779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:41:58.068050621Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=info ts=2026-05-07T16:41:59.748694975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:41:59.754319876Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:42:00.420964914Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-07T16:42:00.73634917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:42:00.73974489Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:42:01.318530395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:42:01.321707292Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:42:01.709807452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:42:01.719148863Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:42:01.795802121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:03.795923034Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-07T16:42:04.83569657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:42:04.838950933Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
ts=2026-05-07T16:42:05.687828521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:42:06.503170469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:42:06.505878816Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:42:06.795285965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:42:06.795316094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:42:06.795324463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:42:06.795339539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:07.171283311Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-07T16:42:08.219263298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:42:08.221010623Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:42:08.354305969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:42:08.357684409Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:42:10.545626941Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-07T16:42:10.805542876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:42:10.809238221Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-07T16:42:11.795048355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:13.920850194Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T16:42:16.795733154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:17.296682861Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-07T16:42:18.674216254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:42:18.680554627Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:42:20.670702797Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T16:42:21.795336001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:42:22.486999346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:42:22.491429949Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:42:22.491454351Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:42:22.491480491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:42:22.494783718Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:42:23.185914774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:42:23.189666664Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:42:23.189694824Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:42:23.189719811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:42:23.192401956Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:42:23.374989847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:42:23.380008653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:42:24.046041107Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-07T16:42:24.121349099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:42:26.342938315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:42:26.346264266Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:42:26.795213464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:27.421055203Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-07T16:42:28.422426763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:42:28.425989569Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:42:30.796274871Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:42:31.050940464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:42:31.054830841Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:42:31.795300601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:42:32.081331726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:42:32.084525767Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:42:32.658483327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:42:32.661587043Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:42:33.923300871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:42:33.926496085Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:42:34.171454038Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-07T16:42:35.070912048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:42:35.074361661Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
ts=2026-05-07T16:42:35.690232126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=info ts=2026-05-07T16:42:36.757313956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:42:36.761241867Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:42:36.777158065Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:42:36.777567106Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:42:36.794959814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:42:36.794972523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:36.79499466Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:42:36.795005405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:42:37.546211123Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-07T16:42:37.809167825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:42:37.813083887Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:42:39.846258712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:42:39.849348082Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:42:40.921441748Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=debug ts=2026-05-07T16:42:40.921465411Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=13
level=info ts=2026-05-07T16:42:41.038649108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:42:41.042183457Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:42:41.795646016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:42:43.274817114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:42:43.278124241Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:42:43.601568383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.4 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:42:43.608117662Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:42:44.298379134Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T16:42:44.298402916Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-07T16:42:44.778680033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:42:44.781538425Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:42:46.795488347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:47.671141283Z caller=checkpoint.go:372 msg="writing series" size="939 B"
ts=2026-05-07T16:42:49.281740844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36586"
level=info ts=2026-05-07T16:42:50.597579739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:42:50.611077238Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:42:51.046084283Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-07T16:42:51.795861595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:53.28261988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:42:54.421429827Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=info ts=2026-05-07T16:42:55.704845256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:42:55.708290335Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=warn ts=2026-05-07T16:42:56.795135385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:42:57.505604365Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:42:57.689233582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:42:57.692520728Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:42:57.692541591Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:42:57.692564218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:42:57.695098312Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:42:57.795774869Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-07T16:42:57.940101205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:42:57.94309217Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:42:57.943111513Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:42:57.993452036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:42:57.996328708Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:42:58.983162723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:42:58.987872072Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:42:59.638570578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:42:59.646346881Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:43:01.170774962Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-07T16:43:01.79566712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:43:04.112271454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:43:04.11607128Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:43:04.546618893Z caller=checkpoint.go:372 msg="writing series" size="558 kB"
level=debug ts=2026-05-07T16:43:04.546666684Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-07T16:43:05.069877509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35736"
ts=2026-05-07T16:43:05.692116446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nb6zc-6526f61a 10.129.0.76:7946"
level=info ts=2026-05-07T16:43:06.171599962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:43:06.17599377Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:43:06.795074672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:43:06.795107681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:43:06.795115723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:43:06.795133037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:07.921249173Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T16:43:08.181130635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:43:08.18471918Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:43:08.212699652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:43:08.215543361Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:43:09.610453501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:43:09.613811699Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:43:09.731059346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:43:09.734360159Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=debug ts=2026-05-07T16:43:11.296461047Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-07T16:43:11.795394797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:43:12.44444364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:43:12.449522495Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:43:13.788524095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:43:13.792425011Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:43:14.671195745Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
ts=2026-05-07T16:43:16.118442376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42170"
level=info ts=2026-05-07T16:43:16.40689515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:43:16.410335856Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-07T16:43:16.795854086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:18.045689466Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-07T16:43:20.621939478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:43:20.627519214Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:43:20.627540926Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:20.627566734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:43:20.630186709Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:43:21.42066404Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-07T16:43:21.794911032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:22.397046273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:43:23.071110831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:43:23.074284101Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:43:23.074304861Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:23.074325886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:43:23.076662244Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:43:24.695145977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:43:24.699691268Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:43:24.79628234Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-07T16:43:25.239353117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:43:25.242484329Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:43:25.49630129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:43:25.499139199Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:43:25.499159722Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:25.499181201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:43:25.501499272Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-07T16:43:25.527582065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:43:25.530470326Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:43:26.45875304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:43:26.462186293Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:43:26.795254642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:28.170995888Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-07T16:43:28.508941296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:43:28.512766781Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:43:29.15007962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:43:31.545915473Z caller=checkpoint.go:372 msg="writing series" size="821 B"
level=warn ts=2026-05-07T16:43:31.795103913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:43:32.910880781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:43:32.915104159Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:43:34.181567594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:43:34.184801374Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:43:34.920931419Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-07T16:43:34.935004049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:43:34.938040813Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:43:35.694765157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:43:35.856318509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:43:35.859610287Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-07T16:43:36.453369287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:43:36.456343093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:43:36.776513013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=info ts=2026-05-07T16:43:36.777573975Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T16:43:36.794951056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:43:36.794978852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:43:36.794998322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:43:36.795046498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:43:37.855822337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:43:37.859121225Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:43:38.297052939Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=debug ts=2026-05-07T16:43:38.297077031Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-07T16:43:38.958018598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.4 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:43:38.964801851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:43:38.964827166Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:38.964861982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.4 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:43:38.971854904Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:43:39.197693752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:43:39.201806912Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:43:41.67111387Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-07T16:43:41.795837595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:45.046426279Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
ts=2026-05-07T16:43:46.120898547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59664"
level=warn ts=2026-05-07T16:43:46.795783195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:47.494359511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:43:48.223656716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:43:48.229343357Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:43:48.42097711Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:43:48.528254763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:43:48.531426849Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:43:49.283919161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:43:49.287809541Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:43:49.287829862Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:49.287852428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:43:49.290916607Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:43:50.354927147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.4 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:43:50.361918647Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:43:50.401908459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:43:50.40639328Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:43:51.157028179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:43:51.161307237Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=warn ts=2026-05-07T16:43:51.7950123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:43:51.796183872Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-07T16:43:53.198341326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:43:53.202365631Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:43:53.202386757Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:53.20241054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:43:53.205484008Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:43:55.171161751Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-07T16:43:56.508513089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:43:56.517967339Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:43:56.795192236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:43:57.311843347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:43:57.314794108Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:43:58.546809031Z caller=checkpoint.go:372 msg="writing series" size="670 kB"
level=debug ts=2026-05-07T16:43:58.546830432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-07T16:43:58.799628452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:43:58.802608859Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:43:58.802628174Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:43:58.802671483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:43:58.804993305Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:43:59.973739566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:43:59.977243006Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
level=info ts=2026-05-07T16:44:00.235583492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:44:00.238661769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:44:01.795099082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:01.921741055Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=info ts=2026-05-07T16:44:03.765241632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:44:03.76906497Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:44:04.361759677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:44:04.364746079Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-05-07T16:44:05.07324981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59692"
level=debug ts=2026-05-07T16:44:05.29624476Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-07T16:44:05.697254448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:44:06.795015018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:44:06.795048804Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:44:06.795060162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:44:06.795058138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:44:06.994096783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:44:06.997144178Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:44:08.608549058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:44:08.612211946Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:44:08.671614153Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=info ts=2026-05-07T16:44:09.048686153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:44:09.052216554Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:44:09.604886281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:44:09.609354848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:44:11.795040529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:12.046151651Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-07T16:44:12.840371409Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:44:13.079096505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:44:13.082654687Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:44:15.420947747Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-07T16:44:16.044299404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:44:16.047437822Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:44:16.065528748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:44:16.069036175Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:44:16.123381378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45720"
level=warn ts=2026-05-07T16:44:16.79518284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:18.796326028Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T16:44:19.286466671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53394"
level=info ts=2026-05-07T16:44:20.870229721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:44:20.875122934Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=warn ts=2026-05-07T16:44:21.795571825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:22.17126634Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=debug ts=2026-05-07T16:44:22.171289573Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-07T16:44:23.306835069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:44:23.408292629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:44:23.411514349Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:44:24.538868487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.4 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:44:24.545374514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:44:24.981650717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:44:24.985138972Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:44:25.232695127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:44:25.235928202Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:44:25.545832992Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-07T16:44:25.621008733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:44:25.624392045Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:44:25.64243198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:44:25.645292637Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:44:25.645307678Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:44:25.645330487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:44:25.647877665Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:44:26.795739322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:44:28.388427454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:44:28.393403772Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=debug ts=2026-05-07T16:44:28.921102358Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-07T16:44:31.795170593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:44:32.176181138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:44:32.179330861Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-07T16:44:32.29574681Z caller=checkpoint.go:372 msg="writing series" size="433 kB"
level=info ts=2026-05-07T16:44:35.163976276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:44:35.16759868Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
level=debug ts=2026-05-07T16:44:35.672103079Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:44:35.67212424Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
ts=2026-05-07T16:44:35.699242568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f794a9ab 10.129.0.80:7946"
level=info ts=2026-05-07T16:44:36.777114496Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:44:36.777514214Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.81:11211 count=1
level=warn ts=2026-05-07T16:44:36.794827365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:44:36.79482924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:44:36.794851216Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:44:36.794858532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:44:37.015922907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:44:37.019526737Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:44:38.084681308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:44:38.087846286Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02f1c83e%3A19e02f1c83f%3A46cdb5a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:44:38.928953052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:44:38.932376725Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-07T16:44:39.046014087Z caller=checkpoint.go:372 msg="writing series" size="571 kB"
level=info ts=2026-05-07T16:44:40.136108574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:44:40.139595422Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-07T16:44:41.79558318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:42.421085742Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-07T16:44:44.056397406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:44:44.061006254Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:44:44.487188137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:44:44.490194897Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02f1a064%3A19e02f1c83f%3Ac6ed90b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:44:45.389467365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.8 MB" total_uncomp="280 MB" avg_uncomp="23 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:44:45.3962304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02f0d7bc%3A19e03051fd6%3A64874721?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:44:45.796341337Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-07T16:44:46.79541636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:48.251860276Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:44:49.171241215Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-07T16:44:49.257366977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=165e6694ba656918 immediate=false num_chunks=1 total_comp="215 kB" avg_comp="215 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"build-e2e-xczp\"}"
level=error ts=2026-05-07T16:44:49.260767126Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/165e6694ba656918/19e03060ecf%3A19e030ec377%3A15728a4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xczp\"}"
ts=2026-05-07T16:44:49.288747746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54760"
level=info ts=2026-05-07T16:44:49.987281442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="435 kB" avg_comp="435 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:44:49.990902588Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02f0d0e3%3A19e032cf896%3A31112613?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:44:50.689573995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c944eb01080eff4a immediate=false num_chunks=1 total_comp="197 kB" avg_comp="197 kB" total_uncomp="887 kB" avg_uncomp="887 kB" full=1 labels="{stream=\"stat-rep-owta\"}"
level=error ts=2026-05-07T16:44:50.692816548Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c944eb01080eff4a/19e0306551f%3A19e030cce35%3A79588984?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-owta\"}"
level=warn ts=2026-05-07T16:44:51.795315544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:44:52.546005136Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:44:55.920855857Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-05-07T16:44:56.79487734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:44:59.26544345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-07T16:44:59.269526013Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e02f1a064%3A19e030f733f%3Aeb5bcc56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-07T16:44:59.295770301Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-07T16:45:01.795378428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:45:01.836455069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="442 kB" avg_uncomp="442 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-07T16:45:01.839729029Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e02f1b5cb%3A19e02f23eaf%3A3be1ca02?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-07T16:45:02.67124299Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-05-07T16:45:05.07697049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57316"
level=info ts=2026-05-07T16:45:05.312438076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="120 kB" avg_uncomp="120 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:45:05.316344342Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02f0f3d6%3A19e02f36e1e%3A299b227f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:45:05.701183836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g6dgq-3df49f41 10.129.0.77:7946"
level=debug ts=2026-05-07T16:45:06.047022057Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:45:06.047045283Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=9
level=info ts=2026-05-07T16:45:06.341019593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="16 MB" avg_uncomp="8.0 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:45:06.345513467Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02f0d15c%3A19e03014ce5%3Aea8f1c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:45:06.795293809Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:45:06.795322428Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:45:06.795330829Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:45:06.795378103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:45:07.121551261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-07T16:45:07.125961728Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e02f0d015%3A19e03493e08%3A1a5d3d2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-07T16:45:08.009278744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="680 kB" total_uncomp="8.8 MB" avg_uncomp="4.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:45:08.013340774Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02f1a064%3A19e030d5820%3A1ccc1aea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:45:08.105593092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.4 MB" total_uncomp="167 MB" avg_uncomp="21 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:45:08.112029822Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02f0cef1%3A19e02f1c6ec%3A34661a9c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:45:09.421582872Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T16:45:09.421605295Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-07T16:45:10.774988222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:45:10.77854435Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:45:10.77856426Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:45:10.778585971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:45:10.781235134Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=warn ts=2026-05-07T16:45:11.795334332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:45:12.796589842Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T16:45:14.031865278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:45:14.035249676Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02f1c83e%3A19e02f1c83f%3Af54cc2d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:45:14.171699441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:45:14.175140694Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0304bd0f%3A19e03060ae8%3A584fea4f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:45:14.677311308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:45:16.17094922Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:45:16.795313695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:45:17.585441634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:45:17.644482266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:45:17.649196711Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:45:17.649215864Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:45:17.649238724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:45:17.65356341Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=info ts=2026-05-07T16:45:17.697781057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9fb869ef9abea367 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-czyh\"}"
level=error ts=2026-05-07T16:45:17.700735117Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9fb869ef9abea367/19e03043a0e%3A19e030ea6de%3A41488ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-czyh\"}"
ts=2026-05-07T16:45:19.291396406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52278"
level=debug ts=2026-05-07T16:45:19.546178702Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-07T16:45:21.7950498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:45:22.68059231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="340 kB" avg_comp="340 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:45:22.684742296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02f1b5cb%3A19e0300e0c7%3A63c0e730?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:45:22.744782478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:45:22.748125444Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02f1a064%3A19e02f1c83f%3Adb7a384f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:45:22.92089563Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T16:45:23.037317937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-07T16:45:23.040484058Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e02f1a064%3A19e02f1c83f%3A3c77fccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-07T16:45:23.940967466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9efeb521262488 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="338 kB" avg_uncomp="338 kB" full=1 labels="{stream=\"integration1-nmqt\"}"
level=error ts=2026-05-07T16:45:23.944292016Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9efeb521262488/19e0304ab57%3A19e030b2997%3Af97320dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-nmqt\"}"
level=debug ts=2026-05-07T16:45:26.296473301Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-07T16:45:26.79536359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:45:29.671578792Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:45:30.237986254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="9.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:45:30.244517997Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02f0ce55%3A19e0319c5b0%3Adce94fa1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:45:31.688848807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:45:31.692666563Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:45:31.692688037Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:45:31.692712087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:45:31.695469431Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02f1c83e%3A19e02f1c83f%3Ae07d6696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:45:31.720573825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca90364dc7b4dc9d immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-wgnh\"}"
level=error ts=2026-05-07T16:45:31.725853515Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca90364dc7b4dc9d/19e0304b327%3A19e031cee48%3A33e10fd0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wgnh\"}"
level=warn ts=2026-05-07T16:45:31.795545944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:45:33.045935325Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:45:33.458499209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="391 kB" avg_comp="391 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-07T16:45:33.462338394Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e02f0d041%3A19e033373e8%3Aa8cdf7c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"